change strange usage of resetting positions in Script

Some fixes to the .pythonrc.py code

.coveragerc

@@ -0,0 +1,18 @@
+[run]
+omit =
+    jedi/_compatibility.py
+
+[report]
+# Regexes for lines to exclude from consideration
+exclude_lines =
+    # Don't complain about missing debug-only code:
+    def __repr__
+    if self\.debug
+
+    # Don't complain if tests don't hit defensive assertion code:
+    raise AssertionError
+    raise NotImplementedError
+
+    # Don't complain if non-runnable code isn't run:
+    if 0:
+    if __name__ == .__main__.:

.gitignore

@@ -1,7 +1,13 @@
+*~
 *.swp
 *.swo
-.ropeproject
 *.pyc
+.ropeproject
+.tox
+.coveralls.yml
+.coverage
 /build/
+/docs/_build/
 /dist/
 jedi.egg-info/
+record.json

.travis.yml

@@ -1,10 +1,21 @@
 language: python
-python:
-    - 2.5
-    - 2.6
-    - 2.7
-    - 3.2
-install: echo "No external requirements"
+env:
+    - TOXENV=py26
+    - TOXENV=py27
+    - TOXENV=py32
+    - TOXENV=py33
+    - TOXENV=cov
+    - TOXENV=sith
+matrix:
+    allow_failures:
+        - env: TOXENV=cov
+        - env: TOXENV=sith
+install:
+    - pip install --quiet --use-mirrors tox
 script:
-    - cd test
-    - ./test.sh
+    - tox
+after_script:
+    - if [ $TOXENV == "cov" ]; then
+      pip install --quiet --use-mirrors coveralls;
+      coveralls;
+      fi

AUTHORS.txt

@@ -1,13 +1,25 @@
 Main Authors
 ============
 
-David Halter (@davidhalter)
+David Halter (@davidhalter) <davidhalter88@gmail.com>
+Takafumi Arakaki (@tkf) <aka.tkf@gmail.com>
 
+Code Contributors
+=================
 
-Contributors
-============
-
-Danilo Bargen (@dbrgn)
+Danilo Bargen (@dbrgn) <gezuru@gmail.com>
+Laurens Van Houtven (@lvh) <_@lvh.cc>
+Aldo Stracquadanio (@Astrac) <aldo.strac@gmail.com>
+Jean-Louis Fuchs (@ganwell) <ganwell@fangorn.ch>
 tek (@tek)
-Takafumi Arakaki (@tkf)
-Aaron Griffin
+Yasha Borevich (@jjay) <j.borevich@gmail.com>
+Aaron Griffin <aaronmgriffin@gmail.com>
+andviro (@andviro)
+Mike Gilbert (@floppym) <floppym@gentoo.org>
+Aaron Meurer (@asmeurer) <asmeurer@gmail.com>
+Lubos Trilety <ltrilety@redhat.com>
+Akinori Hattori (@hattya)
+srusskih (@srusskih)
+
+
+Note: (@user) means a github user name.

CHANGELOG.rst

@@ -0,0 +1,22 @@
+.. :changelog:
+
+Changelog
+---------
+
+0.7.0 (2013-08-09)
+++++++++++++++++++
+* switched from LGPL to MIT license
+* added an Interpreter class to the API to make autocompletion in REPL possible.
+* added autocompletion support for namespace packages
+* add sith.py, a new random testing method
+
+0.6.0 (2013-05-14)
+++++++++++++++++++
+
+* much faster parser with builtin part caching
+* a test suite, thanks @tkf
+
+0.5 versions (2012)
++++++++++++++++++++
+
+* Initial development

CONTRIBUTING.md

@@ -1,11 +1,28 @@
-Pull Requests are great (on the **dev** branch)!
+Pull Requests are great (on the **dev** branch)! Readme/Documentation changes
+are ok in the master branch.
 
  1. Fork the Repo on github.
  2. If you are adding functionality or fixing a bug, please add a test!
- 3. Push to your fork and submit a **pull request to the dev branch**.
+ 3. Add your name to AUTHORS.txt
+ 4. Push to your fork and submit a **pull request to the dev branch**.
 
 My **master** branch is a 100% stable (should be). I only push to it after I am
 certain that things are working out. Many people are using Jedi directly from
-the github master branch. 
+the github master branch.
 
-Please use Pep8 to style your code.
+**Please use PEP8 to style your code.**
+
+
+Changing Issues to Pull Requests (Github)
+-----------------------------------------
+
+If you have have previously filed a GitHub issue and want to contribute code
+that addresses that issue, we prefer it if you use
+[hub](https://github.com/github/hub) to convert your existing issue to a pull
+request. To do that, first push the changes to a separate branch in your fork
+and then issue the following command:
+
+    hub pull-request -b davidhalter:dev -i <issue-number> -h <your-github-username>:<your-branch-name>
+
+It's no strict requirement though, if you don't have hub installed or prefer to
+use the web interface, then feel free to post a traditional pull request.

LICENSE.txt

@@ -1,170 +1,21 @@
-Licensed under the GNU LGPL v3 or later.
-Copyright (C) 2012 David Halter <davidhalter88@gmail.com>.
+The MIT License (MIT)
 
-===============================================================================
+Copyright (c) <2013> <David Halter and others, see AUTHORS.txt>
 
-                   GNU LESSER GENERAL PUBLIC LICENSE
-                       Version 3, 29 June 2007
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
 
- Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/>
- Everyone is permitted to copy and distribute verbatim copies
- of this license document, but changing it is not allowed.
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
 
-
-  This version of the GNU Lesser General Public License incorporates
-the terms and conditions of version 3 of the GNU General Public
-License, supplemented by the additional permissions listed below.
-
-  0. Additional Definitions.
-
-  As used herein, "this License" refers to version 3 of the GNU Lesser
-General Public License, and the "GNU GPL" refers to version 3 of the GNU
-General Public License.
-
-  "The Library" refers to a covered work governed by this License,
-other than an Application or a Combined Work as defined below.
-
-  An "Application" is any work that makes use of an interface provided
-by the Library, but which is not otherwise based on the Library.
-Defining a subclass of a class defined by the Library is deemed a mode
-of using an interface provided by the Library.
-
-  A "Combined Work" is a work produced by combining or linking an
-Application with the Library.  The particular version of the Library
-with which the Combined Work was made is also called the "Linked
-Version".
-
-  The "Minimal Corresponding Source" for a Combined Work means the
-Corresponding Source for the Combined Work, excluding any source code
-for portions of the Combined Work that, considered in isolation, are
-based on the Application, and not on the Linked Version.
-
-  The "Corresponding Application Code" for a Combined Work means the
-object code and/or source code for the Application, including any data
-and utility programs needed for reproducing the Combined Work from the
-Application, but excluding the System Libraries of the Combined Work.
-
-  1. Exception to Section 3 of the GNU GPL.
-
-  You may convey a covered work under sections 3 and 4 of this License
-without being bound by section 3 of the GNU GPL.
-
-  2. Conveying Modified Versions.
-
-  If you modify a copy of the Library, and, in your modifications, a
-facility refers to a function or data to be supplied by an Application
-that uses the facility (other than as an argument passed when the
-facility is invoked), then you may convey a copy of the modified
-version:
-
-   a) under this License, provided that you make a good faith effort to
-   ensure that, in the event an Application does not supply the
-   function or data, the facility still operates, and performs
-   whatever part of its purpose remains meaningful, or
-
-   b) under the GNU GPL, with none of the additional permissions of
-   this License applicable to that copy.
-
-  3. Object Code Incorporating Material from Library Header Files.
-
-  The object code form of an Application may incorporate material from
-a header file that is part of the Library.  You may convey such object
-code under terms of your choice, provided that, if the incorporated
-material is not limited to numerical parameters, data structure
-layouts and accessors, or small macros, inline functions and templates
-(ten or fewer lines in length), you do both of the following:
-
-   a) Give prominent notice with each copy of the object code that the
-   Library is used in it and that the Library and its use are
-   covered by this License.
-
-   b) Accompany the object code with a copy of the GNU GPL and this license
-   document.
-
-  4. Combined Works.
-
-  You may convey a Combined Work under terms of your choice that,
-taken together, effectively do not restrict modification of the
-portions of the Library contained in the Combined Work and reverse
-engineering for debugging such modifications, if you also do each of
-the following:
-
-   a) Give prominent notice with each copy of the Combined Work that
-   the Library is used in it and that the Library and its use are
-   covered by this License.
-
-   b) Accompany the Combined Work with a copy of the GNU GPL and this license
-   document.
-
-   c) For a Combined Work that displays copyright notices during
-   execution, include the copyright notice for the Library among
-   these notices, as well as a reference directing the user to the
-   copies of the GNU GPL and this license document.
-
-   d) Do one of the following:
-
-       0) Convey the Minimal Corresponding Source under the terms of this
-       License, and the Corresponding Application Code in a form
-       suitable for, and under terms that permit, the user to
-       recombine or relink the Application with a modified version of
-       the Linked Version to produce a modified Combined Work, in the
-       manner specified by section 6 of the GNU GPL for conveying
-       Corresponding Source.
-
-       1) Use a suitable shared library mechanism for linking with the
-       Library.  A suitable mechanism is one that (a) uses at run time
-       a copy of the Library already present on the user's computer
-       system, and (b) will operate properly with a modified version
-       of the Library that is interface-compatible with the Linked
-       Version.
-
-   e) Provide Installation Information, but only if you would otherwise
-   be required to provide such information under section 6 of the
-   GNU GPL, and only to the extent that such information is
-   necessary to install and execute a modified version of the
-   Combined Work produced by recombining or relinking the
-   Application with a modified version of the Linked Version. (If
-   you use option 4d0, the Installation Information must accompany
-   the Minimal Corresponding Source and Corresponding Application
-   Code. If you use option 4d1, you must provide the Installation
-   Information in the manner specified by section 6 of the GNU GPL
-   for conveying Corresponding Source.)
-
-  5. Combined Libraries.
-
-  You may place library facilities that are a work based on the
-Library side by side in a single library together with other library
-facilities that are not Applications and are not covered by this
-License, and convey such a combined library under terms of your
-choice, if you do both of the following:
-
-   a) Accompany the combined library with a copy of the same work based
-   on the Library, uncombined with any other library facilities,
-   conveyed under the terms of this License.
-
-   b) Give prominent notice with the combined library that part of it
-   is a work based on the Library, and explaining where to find the
-   accompanying uncombined form of the same work.
-
-  6. Revised Versions of the GNU Lesser General Public License.
-
-  The Free Software Foundation may publish revised and/or new versions
-of the GNU Lesser General Public License from time to time. Such new
-versions will be similar in spirit to the present version, but may
-differ in detail to address new problems or concerns.
-
-  Each version is given a distinguishing version number. If the
-Library as you received it specifies that a certain numbered version
-of the GNU Lesser General Public License "or any later version"
-applies to it, you have the option of following the terms and
-conditions either of that published version or of any later version
-published by the Free Software Foundation. If the Library as you
-received it does not specify a version number of the GNU Lesser
-General Public License, you may choose any version of the GNU Lesser
-General Public License ever published by the Free Software Foundation.
-
-  If the Library as you received it specifies that a proxy can decide
-whether future versions of the GNU Lesser General Public License shall
-apply, that proxy's public statement of acceptance of any version is
-permanent authorization for you to choose that version for the
-Library.
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

MANIFEST.in

@@ -1,5 +1,12 @@
 include README.rst
+include CHANGELOG.rst
 include LICENSE.txt
 include AUTHORS.txt
-include jedi/mixin/*.py
+include .coveragerc
+include sith.py
+include conftest.py
+include pytest.ini
+include tox.ini
+include jedi/mixin/*.pym
+recursive-include test *
 recursive-exclude * *.pyc

README.rst

@@ -1,262 +1,136 @@
-########################################
-Jedi - an awesome Python auto-completion
-########################################
+###################################################
+Jedi - an awesome autocompletion library for Python
+###################################################
 
 .. image:: https://secure.travis-ci.org/davidhalter/jedi.png?branch=master
     :target: http://travis-ci.org/davidhalter/jedi
     :alt: Travis-CI build status
 
-**beta testing**
+.. image:: https://coveralls.io/repos/davidhalter/jedi/badge.png?branch=master
+    :target: https://coveralls.io/r/davidhalter/jedi
+    :alt: Coverage Status
 
-*If you have any comments or feature requests, please tell me! I really want to
-know, what you think about Jedi.*
+.. image:: https://pypip.in/d/jedi/badge.png
+        :target: https://crate.io/packages/jedi/
 
-Jedi is an autocompletion tool for Python. It works. With and without syntax
-errors. Sometimes it sucks, but that's normal in dynamic languages. But it
-sucks less than other tools. It understands almost all of the basic Python
-syntax elements including many builtins.
+Jedi is an autocompletion tool for Python that can be used in IDEs/editors.
+Jedi works. Jedi is fast. It understands all of the basic Python syntax
+elements including many builtin functions.
 
-Jedi suports two different goto functions and has support for renaming.
-Probably it will also have some support for refactoring in the future.
+Additionaly, Jedi suports two different goto functions and has support for
+renaming as well as Pydoc support and some other IDE features.
 
-Jedi uses a very simple interface to connect with IDE's. As an reference, there
-is a VIM implementation, which uses Jedi's autocompletion. However, I encourage
-you to use Jedi in your IDEs. Start writing plugins! If there are problems with
-licensing, just contact me.
+Jedi uses a very simple API to connect with IDE's. There's a reference
+implementation as a `VIM-Plugin <https://github.com/davidhalter/jedi-vim>`_,
+which uses Jedi's autocompletion.  I encourage you to use Jedi in your IDEs.
+It's really easy. If there are any problems (also with licensing), just contact
+me.
 
-At the moment Jedi can be used as a 
-`VIM-Plugin <http://github.com/davidhalter/jedi-vim>`_. So, if you want to test
-Jedi for now, you'll have to use VIM. But there are new plugins emerging:
+Jedi can be used with the following editors:
+
+- Vim (jedi-vim_, YouCompleteMe_)
+- Emacs (Jedi.el_)
+- Sublime Text (SublimeJEDI_ [ST2 + ST3], anaconda_ [only ST3])
+
+And it powers the following projects:
+
+- wdb_
 
-- `Emacs-Plugin <https://github.com/tkf/emacs-jedi>`_
-- `Sublime-Plugin <https://github.com/svaiter/SublimeJEDI>`_ **Under construction**
 
 Here are some pictures:
 
-.. image:: https://github.com/davidhalter/jedi/raw/master/screenshot_complete.png
+.. image:: https://github.com/davidhalter/jedi/raw/master/docs/_screenshots/screenshot_complete.png
 
 Completion for almost anything (Ctrl+Space).
 
-.. image:: https://github.com/davidhalter/jedi/raw/master/screenshot_function.png
+.. image:: https://github.com/davidhalter/jedi/raw/master/docs/_screenshots/screenshot_function.png
 
 Display of function/class bodies, docstrings.
 
-.. image:: https://github.com/davidhalter/jedi/raw/master/screenshot_pydoc.png
+.. image:: https://github.com/davidhalter/jedi/raw/master/docs/_screenshots/screenshot_pydoc.png
 
 Pydoc support (with highlighting, Shift+k).
 
 There is also support for goto and renaming.
 
-Get the latest from `github <http://github.com/davidhalter/jedi>`_.
+Get the latest version from `github <https://github.com/davidhalter/jedi>`_
+(master branch should always be kind of stable/working).
+
+Docs are available at `https://jedi.readthedocs.org/
+<https://jedi.readthedocs.org/>`_. Pull requests with documentation
+enhancements and/or fixes are awesome and most welcome. Jedi uses `semantic
+versioning <http://semver.org/>`_.
 
 
 Installation
 ============
 
-You can either include Jedi as a submodule in your text editor plugin (like
-jedi-vim_ does it by default), or you
-can install Jedi systemwide.
-
-The preferred way to install the Jedi library into your system is by using
-pip_::
-
-    sudo pip install jedi
-
-If you want to install the current development version::
-
-    sudo pip install -e git://github.com/davidhalter/jedi.git#egg=jedi
+    pip install jedi
 
 Note: This just installs the Jedi library, not the editor plugins. For
 information about how to make it work with your editor, refer to the
 corresponding documentation.
 
+You don't want to use ``pip``? Please refer to the `manual
+<https://jedi.readthedocs.org/en/latest/docs/installation.html>`_.
 
-Support
+
+Feature Support and Caveats
+===========================
+
+Jedi really understands your Python code. For a comprehensive list what Jedi
+can do, see: `Features
+<https://jedi.readthedocs.org/en/latest/docs/features.html>`_. A list of
+caveats can be found on the same page.
+
+You can run Jedi on cPython 2.6, 2.7, 3.2 or 3.3, but it should also
+understand/parse code older than those versions.
+
+Tips on how to use Jedi efficiently can be found `here
+<https://jedi.readthedocs.org/en/latest/docs/recipes.html>`_.
+
+
+API for IDEs
+============
+
+It's very easy to create an editor plugin that uses Jedi. See `Plugin API
+<https://jedi.readthedocs.org/en/latest/docs/plugin-api.html>`_ for more
+information.
+
+
+Development
+===========
+
+There's a pretty good and extensive `development documentation
+<https://jedi.readthedocs.org/en/latest/docs/development.html>`_.
+
+
+Testing
 =======
 
-Jedi supports Python 2.5 up to 3.x. There is just one code base, for both
-Python 2 and 3.
-Jedi supports many of the widely used Python features:
+The test suite depends on ``tox`` and ``pytest``::
 
-- builtin functions/classes support
-- complex module / function / class structures
-- ignores syntax and indentation errors
-- multiple returns / yields
-- tuple assignments / array indexing / dictionary indexing
-- exceptions / with-statement
-- \*args / \*\*kwargs
-- decorators
-- descriptors -> property / staticmethod / classmethod
-- closures
-- generators (yield statement) / iterators
-- support for some magic methods: ``__call__``, ``__iter__``, ``__next__``,
-  ``__get__``, ``__getitem__``, ``__init__``
-- support for list.append, set.add, list.extend, etc.
-- (nested) list comprehensions / ternary expressions
-- relative imports
-- ``getattr()`` / ``__getattr__`` / ``__getattribute__``
-- function annotations (py3k feature, are ignored right now, but being parsed.
-  I don't know what to do with them.)
-- class decorators (py3k feature, are being ignored too, until I find a use
-  case, that doesn't work with Jedi)
-- simple/usual ``sys.path`` modifications
-- ``isinstance`` checks for if/while/assert
-- virtualenv support
-- infer function arguments with sphinx (and other) docstrings
+    pip install tox pytest
 
-However, it does not yet support (and probably will in future versions, because
-they are on my todo list):
+To run the tests for all supported Python versions::
 
-- manipulations of instances outside the instance variables, without using
-  functions
+    tox
 
-It does not support (and most probably will not in future versions): 
+If you want to test only a specific Python version (e.g. Python 2.7), it's as
+easy as ::
 
-- metaclasses (how could an auto-completion ever support this)
-- ``setattr()``, ``__import__()``
-- Writing to some dicts: ``globals()``, ``locals()``, ``object.__dict__``
-- evaluate ``if`` / ``while``
+    tox -e py27
+
+Tests are also run automatically on `Travis CI
+<https://travis-ci.org/davidhalter/jedi/>`_.
+
+For more detailed information visit the `testing documentation
+<https://jedi.readthedocs.org/en/latest/docs/testing.html>`_
 
 
-Caveats
-=======
-
-This framework should work for both Python 2/3. However, some things were just
-not as *pythonic* in Python 2 as things should be. To keep things simple, some
-things have been held back:
-
-- Classes: Always Python 3 like, therefore all classes inherit from ``object``.
-- Generators: No ``next`` method. The ``__next__`` method is used instead.
-- Exceptions are only looked at in the form of ``Exception as e``, no comma!
-
-Syntax errors and other strange stuff, that is defined differently in the
-Python language, may lead to undefined behaviour of the completion. Jedi is
-**NOT** a Python compiler, that tries to correct you. It is a tool that wants
-to help you. But **YOU** have to know Python, not Jedi.
-
-Importing ``numpy`` can be quite slow sometimes, as well as loading the builtins
-the first time. If you want to speed it up, you could write import hooks in
-jedi, which preloads this stuff. However, once loaded, this is not a problem
-anymore. The same is true for huge modules like ``PySide``, ``wx``, etc.
-
-Security is an important issue for Jedi. Therefore no Python code is executed.
-As long as you write pure python, everything is evaluated statically. But: If
-you use builtin modules (`c_builtin`) there is no other option than to execute
-those modules. However: Execute isn't that critical (as e.g. in pythoncomplete,
-which used to execute *every* import!), because it means one import and no
-more. So basically the only dangerous thing is using the import itself. If your
-`c_builtin` uses some strange initializations, it might be dangerous. But if it
-does you're screwed anyways, because eventualy you're going to execute your
-code, which executes the import.
-
-
-A little history
-================
-
-The Star Wars Jedi are awesome. My Jedi software tries to imitate a little bit
-of the precognition the Jedi have. There is even an awesome `scene
-<http://www.youtube.com/watch?v=5BDO3pyavOY>`_ of Monty Python Jedi's :-). 
-
-But actually the name hasn't so much to do with Star Wars. It's part of my
-second name.
-
-After I explained Guido van Rossum, how some parts of my auto-completion work,
-he said (we drank a beer or two):
-
-    *Oh, that worries me*
-
-When it's finished, I hope he'll like it :-)
-
-I actually started Jedi, because there were no good solutions available for
-VIM. Most auto-completions just didn't work well. The only good solution was
-PyCharm. I just like my good old VIM. Rope was never really intended to be an
-auto-completion (and also I really hate project folders for my Python scripts).
-It's more of a refactoring suite. So I decided to do my own version of a
-completion, which would execute non-dangerous code. But I soon realized, that
-this wouldn't work. So I built an extremely recursive thing which understands
-many of Python's key features.
-
-By the way, I really tried to program it as understandable as possible. But I
-think understanding it might need quite some time, because of its recursive
-nature.
-
-
-API-Design for IDEs
-===================
-
-If you want to set up an IDE with Jedi, you need to ``import jedi``. You should
-have the following objects available:
-
-::
-
-    Script(source, line, column, source_path)
-
-``source`` would be the source of your python file/script, separated by new
-lines. ``line`` is the current line you want to perform actions on (starting
-with line #1 as the first line). ``column`` represents the current
-column/indent of the cursor (starting with zero). ``source_path`` should be the
-path of your file in the file system.
-
-It returns a script object that contains the relevant information for the other
-functions to work without params.
-
-::
-
-    Script().complete
-
-Returns ``api.Completion`` objects. Those objects have got
-informations about the completions. More than just names.
-
-::
-
-    Script().goto
-
-Similar to complete. The returned ``api.Definition`` objects contain
-information about the definitions found.
-
-::
-
-    Script().get_definition
-
-Mostly used for tests. Like goto, but follows statements and imports and
-doesn't break there. You probably don't want to use this function. It's
-mostly for testing.
-
-::
-
-    Script().related_names
-
-Returns all names that point to the definition of the name under the
-cursor. This is also very useful for refactoring (renaming).
-
-::
-
-    Script().get_in_function_call
-
-Get the ``Function`` object of the call you're currently in, e.g.: ``abs(``
-with the cursor at the end would return the builtin ``abs`` function.
-
-::
-
-    NotFoundError
-
-If you use the goto function and no valid identifier (name) is at the
-place of the cursor (position). It will raise this exception.
-
-::
-
-    set_debug_function
-
-Sets a callback function for ``debug.py``. This function is called with
-multiple text objects, in python 3 you could insert ``print``.
-
-::
-
-    settings
-
-Access to the ``settings.py`` module. The settings are described there.
-
-
-
-.. _jedi-vim: http://github.com/davidhalter/jedi-vim
-.. _pip: http://www.pip-installer.org/
+.. _jedi-vim: https://github.com/davidhalter/jedi-vim
+.. _youcompleteme: http://valloric.github.io/YouCompleteMe/
+.. _Jedi.el: https://github.com/tkf/emacs-jedi
+.. _sublimejedi: https://github.com/srusskih/SublimeJEDI
+.. _anaconda: https://github.com/DamnWidget/anaconda
+.. _wdb: https://github.com/Kozea/wdb

conftest.py

@@ -0,0 +1,34 @@
+import tempfile
+import shutil
+
+import jedi
+
+
+collect_ignore = ["setup.py"]
+
+
+# The following hooks (pytest_configure, pytest_unconfigure) are used
+# to modify `jedi.settings.cache_directory` because `clean_jedi_cache`
+# has no effect during doctests.  Without these hooks, doctests uses
+# user's cache (e.g., ~/.cache/jedi/).  We should remove this
+# workaround once the problem is fixed in py.test.
+#
+# See:
+# - https://github.com/davidhalter/jedi/pull/168
+# - https://bitbucket.org/hpk42/pytest/issue/275/
+
+jedi_cache_directory_orig = None
+jedi_cache_directory_temp = None
+
+
+def pytest_configure(config):
+    global jedi_cache_directory_orig, jedi_cache_directory_temp
+    jedi_cache_directory_orig = jedi.settings.cache_directory
+    jedi_cache_directory_temp = tempfile.mkdtemp(prefix='jedi-test-')
+    jedi.settings.cache_directory = jedi_cache_directory_temp
+
+
+def pytest_unconfigure(config):
+    global jedi_cache_directory_orig, jedi_cache_directory_temp
+    jedi.settings.cache_directory = jedi_cache_directory_orig
+    shutil.rmtree(jedi_cache_directory_temp)

docs/Makefile

@@ -0,0 +1,153 @@
+# Makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line.
+SPHINXOPTS    =
+SPHINXBUILD   = sphinx-build
+PAPER         =
+BUILDDIR      = _build
+
+# Internal variables.
+PAPEROPT_a4     = -D latex_paper_size=a4
+PAPEROPT_letter = -D latex_paper_size=letter
+ALLSPHINXOPTS   = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
+# the i18n builder cannot share the environment and doctrees with the others
+I18NSPHINXOPTS  = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
+
+.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext
+
+help:
+	@echo "Please use \`make <target>' where <target> is one of"
+	@echo "  html       to make standalone HTML files"
+	@echo "  dirhtml    to make HTML files named index.html in directories"
+	@echo "  singlehtml to make a single large HTML file"
+	@echo "  pickle     to make pickle files"
+	@echo "  json       to make JSON files"
+	@echo "  htmlhelp   to make HTML files and a HTML help project"
+	@echo "  qthelp     to make HTML files and a qthelp project"
+	@echo "  devhelp    to make HTML files and a Devhelp project"
+	@echo "  epub       to make an epub"
+	@echo "  latex      to make LaTeX files, you can set PAPER=a4 or PAPER=letter"
+	@echo "  latexpdf   to make LaTeX files and run them through pdflatex"
+	@echo "  text       to make text files"
+	@echo "  man        to make manual pages"
+	@echo "  texinfo    to make Texinfo files"
+	@echo "  info       to make Texinfo files and run them through makeinfo"
+	@echo "  gettext    to make PO message catalogs"
+	@echo "  changes    to make an overview of all changed/added/deprecated items"
+	@echo "  linkcheck  to check all external links for integrity"
+	@echo "  doctest    to run all doctests embedded in the documentation (if enabled)"
+
+clean:
+	-rm -rf $(BUILDDIR)/*
+
+html:
+	$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html
+	@echo
+	@echo "Build finished. The HTML pages are in $(BUILDDIR)/html."
+
+dirhtml:
+	$(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml
+	@echo
+	@echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml."
+
+singlehtml:
+	$(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml
+	@echo
+	@echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml."
+
+pickle:
+	$(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle
+	@echo
+	@echo "Build finished; now you can process the pickle files."
+
+json:
+	$(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json
+	@echo
+	@echo "Build finished; now you can process the JSON files."
+
+htmlhelp:
+	$(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp
+	@echo
+	@echo "Build finished; now you can run HTML Help Workshop with the" \
+	      ".hhp project file in $(BUILDDIR)/htmlhelp."
+
+qthelp:
+	$(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp
+	@echo
+	@echo "Build finished; now you can run "qcollectiongenerator" with the" \
+	      ".qhcp project file in $(BUILDDIR)/qthelp, like this:"
+	@echo "# qcollectiongenerator $(BUILDDIR)/qthelp/Jedi.qhcp"
+	@echo "To view the help file:"
+	@echo "# assistant -collectionFile $(BUILDDIR)/qthelp/Jedi.qhc"
+
+devhelp:
+	$(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp
+	@echo
+	@echo "Build finished."
+	@echo "To view the help file:"
+	@echo "# mkdir -p $$HOME/.local/share/devhelp/Jedi"
+	@echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/Jedi"
+	@echo "# devhelp"
+
+epub:
+	$(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub
+	@echo
+	@echo "Build finished. The epub file is in $(BUILDDIR)/epub."
+
+latex:
+	$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
+	@echo
+	@echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex."
+	@echo "Run \`make' in that directory to run these through (pdf)latex" \
+	      "(use \`make latexpdf' here to do that automatically)."
+
+latexpdf:
+	$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
+	@echo "Running LaTeX files through pdflatex..."
+	$(MAKE) -C $(BUILDDIR)/latex all-pdf
+	@echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex."
+
+text:
+	$(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text
+	@echo
+	@echo "Build finished. The text files are in $(BUILDDIR)/text."
+
+man:
+	$(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man
+	@echo
+	@echo "Build finished. The manual pages are in $(BUILDDIR)/man."
+
+texinfo:
+	$(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo
+	@echo
+	@echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo."
+	@echo "Run \`make' in that directory to run these through makeinfo" \
+	      "(use \`make info' here to do that automatically)."
+
+info:
+	$(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo
+	@echo "Running Texinfo files through makeinfo..."
+	make -C $(BUILDDIR)/texinfo info
+	@echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo."
+
+gettext:
+	$(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale
+	@echo
+	@echo "Build finished. The message catalogs are in $(BUILDDIR)/locale."
+
+changes:
+	$(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes
+	@echo
+	@echo "The overview file is in $(BUILDDIR)/changes."
+
+linkcheck:
+	$(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck
+	@echo
+	@echo "Link check complete; look for any errors in the above output " \
+	      "or in $(BUILDDIR)/linkcheck/output.txt."
+
+doctest:
+	$(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest
+	@echo "Testing of doctests in the sources finished, look at the " \
+	      "results in $(BUILDDIR)/doctest/output.txt."

docs/_static/logo-src.txt

@@ -0,0 +1,3 @@
+The source of the logo is a photoshop file hosted here:
+
+https://dl.dropboxusercontent.com/u/170011615/Jedi12_Logo.psd.xz

docs/_templates/ghbuttons.html

@@ -0,0 +1,4 @@
+<h3>Github</h3>
+<iframe src="http://ghbtns.com/github-btn.html?user=davidhalter&repo=jedi&type=watch&count=true&size=large"
+        frameborder="0" scrolling="0" width="170" height="30" allowtransparency="true"></iframe>
+<br><br>

docs/_templates/sidebarlogo.html

@@ -0,0 +1,3 @@
+<p class="logo"><a href="{{ pathto(master_doc) }}">
+  <img class="logo" src="{{ pathto('_static/logo.png', 1) }}" alt="Logo"/>
+</a></p>

docs/_themes/flask/LICENSE

@@ -0,0 +1,37 @@
+Copyright (c) 2010 by Armin Ronacher.
+
+Some rights reserved.
+
+Redistribution and use in source and binary forms of the theme, with or
+without modification, are permitted provided that the following conditions
+are met:
+
+* Redistributions of source code must retain the above copyright
+  notice, this list of conditions and the following disclaimer.
+
+* Redistributions in binary form must reproduce the above
+  copyright notice, this list of conditions and the following
+  disclaimer in the documentation and/or other materials provided
+  with the distribution.
+
+* The names of the contributors may not be used to endorse or
+  promote products derived from this software without specific
+  prior written permission.
+
+We kindly ask you to only use these themes in an unmodified manner just
+for Flask and Flask-related products, not for unrelated projects.  If you
+like the visual style and want to use it for your own projects, please
+consider making some larger changes to the themes (such as changing
+font faces, sizes, colors or margins).
+
+THIS THEME IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ARISING IN ANY WAY OUT OF THE USE OF THIS THEME, EVEN IF ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGE.

docs/_themes/flask/layout.html

@@ -0,0 +1,28 @@
+{%- extends "basic/layout.html" %}
+{%- block extrahead %}
+  {{ super() }}
+  {% if theme_touch_icon %}
+  <link rel="apple-touch-icon" href="{{ pathto('_static/' ~ theme_touch_icon, 1) }}" />
+  {% endif %}
+  <link media="only screen and (max-device-width: 480px)" href="{{
+    pathto('_static/small_flask.css', 1) }}" type= "text/css" rel="stylesheet" />
+  <a href="https://github.com/davidhalter/jedi">
+    <img style="position: absolute; top: 0; right: 0; border: 0;" src="https://s3.amazonaws.com/github/ribbons/forkme_right_red_aa0000.png" alt="Fork me on GitHub">
+  </a>
+{% endblock %}
+{%- block relbar2 %}{% endblock %}
+{% block header %}
+  {{ super() }}
+  {% if pagename == 'index' %}
+  <div class=indexwrapper>
+  {% endif %}
+{% endblock %}
+{%- block footer %}
+  <div class="footer">
+    &copy; Copyright {{ copyright }}.
+    Created using <a href="http://sphinx.pocoo.org/">Sphinx</a>.
+  </div>
+  {% if pagename == 'index' %}
+  </div>
+  {% endif %}
+{%- endblock %}

docs/_themes/flask/relations.html

@@ -0,0 +1,19 @@
+<h3>Related Topics</h3>
+<ul>
+  <li><a href="{{ pathto(master_doc) }}">Documentation overview</a><ul>
+  {%- for parent in parents %}
+  <li><a href="{{ parent.link|e }}">{{ parent.title }}</a><ul>
+  {%- endfor %}
+    {%- if prev %}
+      <li>Previous: <a href="{{ prev.link|e }}" title="{{ _('previous chapter')
+        }}">{{ prev.title }}</a></li>
+    {%- endif %}
+    {%- if next %}
+      <li>Next: <a href="{{ next.link|e }}" title="{{ _('next chapter')
+        }}">{{ next.title }}</a></li>
+    {%- endif %}
+  {%- for parent in parents %}
+  </ul></li>
+  {%- endfor %}
+  </ul></li>
+</ul>

docs/_themes/flask/static/flasky.css_t

@@ -0,0 +1,394 @@
+/*
+ * flasky.css_t
+ * ~~~~~~~~~~~~
+ *
+ * :copyright: Copyright 2010 by Armin Ronacher.
+ * :license: Flask Design License, see LICENSE for details.
+ */
+
+{% set page_width = '940px' %}
+{% set sidebar_width = '220px' %}
+ 
+@import url("basic.css");
+ 
+/* -- page layout ----------------------------------------------------------- */
+ 
+body {
+    font-family: 'Georgia', serif;
+    font-size: 17px;
+    background-color: white;
+    color: #000;
+    margin: 0;
+    padding: 0;
+}
+
+div.document {
+    width: {{ page_width }};
+    margin: 30px auto 0 auto;
+}
+
+div.documentwrapper {
+    float: left;
+    width: 100%;
+}
+
+div.bodywrapper {
+    margin: 0 0 0 {{ sidebar_width }};
+}
+
+div.sphinxsidebar {
+    width: {{ sidebar_width }};
+}
+
+hr {
+    border: 1px solid #B1B4B6;
+}
+ 
+div.body {
+    background-color: #ffffff;
+    color: #3E4349;
+    padding: 0 30px 0 30px;
+}
+
+img.floatingflask {
+    padding: 0 0 10px 10px;
+    float: right;
+}
+ 
+div.footer {
+    width: {{ page_width }};
+    margin: 20px auto 30px auto;
+    font-size: 14px;
+    color: #888;
+    text-align: right;
+}
+
+div.footer a {
+    color: #888;
+}
+
+div.related {
+    display: none;
+}
+ 
+div.sphinxsidebar a {
+    color: #444;
+    text-decoration: none;
+    border-bottom: 1px dotted #999;
+}
+
+div.sphinxsidebar a:hover {
+    border-bottom: 1px solid #999;
+}
+ 
+div.sphinxsidebar {
+    font-size: 14px;
+    line-height: 1.5;
+}
+
+div.sphinxsidebarwrapper {
+    padding: 18px 10px;
+}
+
+div.sphinxsidebarwrapper p.logo {
+    padding: 0 0 20px 0;
+    margin: 0;
+    text-align: center;
+}
+ 
+div.sphinxsidebar h3,
+div.sphinxsidebar h4 {
+    font-family: 'Garamond', 'Georgia', serif;
+    color: #444;
+    font-size: 24px;
+    font-weight: normal;
+    margin: 0 0 5px 0;
+    padding: 0;
+}
+
+div.sphinxsidebar h4 {
+    font-size: 20px;
+}
+ 
+div.sphinxsidebar h3 a {
+    color: #444;
+}
+
+div.sphinxsidebar p.logo a,
+div.sphinxsidebar h3 a,
+div.sphinxsidebar p.logo a:hover,
+div.sphinxsidebar h3 a:hover {
+    border: none;
+}
+ 
+div.sphinxsidebar p {
+    color: #555;
+    margin: 10px 0;
+}
+
+div.sphinxsidebar ul {
+    margin: 10px 0;
+    padding: 0;
+    color: #000;
+}
+ 
+div.sphinxsidebar input {
+    border: 1px solid #ccc;
+    font-family: 'Georgia', serif;
+    font-size: 1em;
+}
+ 
+/* -- body styles ----------------------------------------------------------- */
+ 
+a {
+    color: #004B6B;
+    text-decoration: underline;
+}
+ 
+a:hover {
+    color: #6D4100;
+    text-decoration: underline;
+}
+ 
+div.body h1,
+div.body h2,
+div.body h3,
+div.body h4,
+div.body h5,
+div.body h6 {
+    font-family: 'Garamond', 'Georgia', serif;
+    font-weight: normal;
+    margin: 30px 0px 10px 0px;
+    padding: 0;
+}
+
+{% if theme_index_logo %}
+div.indexwrapper h1 {
+    text-indent: -999999px;
+    background: url({{ theme_index_logo }}) no-repeat center center;
+    height: {{ theme_index_logo_height }};
+}
+{% endif %}
+ 
+div.body h1 { margin-top: 0; padding-top: 0; font-size: 240%; }
+div.body h2 { font-size: 180%; }
+div.body h3 { font-size: 150%; }
+div.body h4 { font-size: 130%; }
+div.body h5 { font-size: 100%; }
+div.body h6 { font-size: 100%; }
+ 
+a.headerlink {
+    color: #ddd;
+    padding: 0 4px;
+    text-decoration: none;
+}
+ 
+a.headerlink:hover {
+    color: #444;
+}
+ 
+div.body p, div.body dd, div.body li {
+    line-height: 1.4em;
+}
+
+div.admonition {
+    background: #fafafa;
+    margin: 20px -30px;
+    padding: 10px 30px;
+    border-top: 1px solid #ccc;
+    border-bottom: 1px solid #ccc;
+}
+
+div.admonition tt.xref, div.admonition a tt {
+    border-bottom: 1px solid #fafafa;
+}
+
+dd div.admonition {
+    margin-left: -60px;
+    padding-left: 60px;
+}
+
+div.admonition p.admonition-title {
+    font-family: 'Garamond', 'Georgia', serif;
+    font-weight: normal;
+    font-size: 24px;
+    margin: 0 0 10px 0;
+    padding: 0;
+    line-height: 1;
+}
+
+div.admonition p.last {
+    margin-bottom: 0;
+}
+
+div.highlight {
+    background-color: white;
+}
+
+dt:target, .highlight {
+    background: #FAF3E8;
+}
+
+div.note {
+    background-color: #eee;
+    border: 1px solid #ccc;
+}
+ 
+div.seealso {
+    background-color: #ffc;
+    border: 1px solid #ff6;
+}
+ 
+div.topic {
+    background-color: #eee;
+}
+ 
+p.admonition-title {
+    display: inline;
+}
+ 
+p.admonition-title:after {
+    content: ":";
+}
+
+pre, tt {
+    font-family: 'Consolas', 'Menlo', 'Deja Vu Sans Mono', 'Bitstream Vera Sans Mono', monospace;
+    font-size: 0.9em;
+}
+
+img.screenshot {
+}
+
+tt.descname, tt.descclassname {
+    font-size: 0.95em;
+}
+
+tt.descname {
+    padding-right: 0.08em;
+}
+
+img.screenshot {
+    -moz-box-shadow: 2px 2px 4px #eee;
+    -webkit-box-shadow: 2px 2px 4px #eee;
+    box-shadow: 2px 2px 4px #eee;
+}
+
+table.docutils {
+    border: 1px solid #888;
+    -moz-box-shadow: 2px 2px 4px #eee;
+    -webkit-box-shadow: 2px 2px 4px #eee;
+    box-shadow: 2px 2px 4px #eee;
+}
+
+table.docutils td, table.docutils th {
+    border: 1px solid #888;
+    padding: 0.25em 0.7em;
+}
+
+table.field-list, table.footnote {
+    border: none;
+    -moz-box-shadow: none;
+    -webkit-box-shadow: none;
+    box-shadow: none;
+}
+
+table.footnote {
+    margin: 15px 0;
+    width: 100%;
+    border: 1px solid #eee;
+    background: #fdfdfd;
+    font-size: 0.9em;
+}
+
+table.footnote + table.footnote {
+    margin-top: -15px;
+    border-top: none;
+}
+
+table.field-list th {
+    padding: 0 0.8em 0 0;
+}
+
+table.field-list td {
+    padding: 0;
+}
+
+table.footnote td.label {
+    width: 0px;
+    padding: 0.3em 0 0.3em 0.5em;
+}
+
+table.footnote td {
+    padding: 0.3em 0.5em;
+}
+
+dl {
+    margin: 0;
+    padding: 0;
+}
+
+dl dd {
+    margin-left: 30px;
+}
+
+blockquote {
+    margin: 0 0 0 30px;
+    padding: 0;
+}
+
+ul, ol {
+    margin: 10px 0 10px 30px;
+    padding: 0;
+}
+ 
+pre {
+    background: #eee;
+    padding: 7px 30px;
+    margin: 15px -30px;
+    line-height: 1.3em;
+}
+
+dl pre, blockquote pre, li pre {
+    margin-left: -60px;
+    padding-left: 60px;
+}
+
+dl dl pre {
+    margin-left: -90px;
+    padding-left: 90px;
+}
+ 
+tt {
+    background-color: #ecf0f3;
+    color: #222;
+    /* padding: 1px 2px; */
+}
+
+tt.xref, a tt {
+    background-color: #FBFBFB;
+    border-bottom: 1px solid white;
+}
+
+a.reference {
+    text-decoration: none;
+    border-bottom: 1px dotted #004B6B;
+}
+
+a.reference:hover {
+    border-bottom: 1px solid #6D4100;
+}
+
+a.footnote-reference {
+    text-decoration: none;
+    font-size: 0.7em;
+    vertical-align: top;
+    border-bottom: 1px dotted #004B6B;
+}
+
+a.footnote-reference:hover {
+    border-bottom: 1px solid #6D4100;
+}
+
+a:hover tt {
+    background: #EEE;
+}

docs/_themes/flask/static/small_flask.css

@@ -0,0 +1,70 @@
+/*
+ * small_flask.css_t
+ * ~~~~~~~~~~~~~~~~~
+ *
+ * :copyright: Copyright 2010 by Armin Ronacher.
+ * :license: Flask Design License, see LICENSE for details.
+ */
+
+body {
+    margin: 0;
+    padding: 20px 30px;
+}
+
+div.documentwrapper {
+    float: none;
+    background: white;
+}
+
+div.sphinxsidebar {
+    display: block;
+    float: none;
+    width: 102.5%;
+    margin: 50px -30px -20px -30px;
+    padding: 10px 20px;
+    background: #333;
+    color: white;
+}
+
+div.sphinxsidebar h3, div.sphinxsidebar h4, div.sphinxsidebar p,
+div.sphinxsidebar h3 a {
+    color: white;
+}
+
+div.sphinxsidebar a {
+    color: #aaa;
+}
+
+div.sphinxsidebar p.logo {
+    display: none;
+}
+
+div.document {
+    width: 100%;
+    margin: 0;
+}
+
+div.related {
+    display: block;
+    margin: 0;
+    padding: 10px 0 20px 0;
+}
+
+div.related ul,
+div.related ul li {
+    margin: 0;
+    padding: 0;
+}
+
+div.footer {
+    display: none;
+}
+
+div.bodywrapper {
+    margin: 0;
+}
+
+div.body {
+    min-height: 0;
+    padding: 0;
+}

docs/_themes/flask/theme.conf

@@ -0,0 +1,9 @@
+[theme]
+inherit = basic
+stylesheet = flasky.css
+pygments_style = flask_theme_support.FlaskyStyle
+
+[options]
+index_logo = 
+index_logo_height = 120px
+touch_icon = 

docs/_themes/flask_theme_support.py

@@ -0,0 +1,125 @@
+"""
+Copyright (c) 2010 by Armin Ronacher.
+
+Some rights reserved.
+
+Redistribution and use in source and binary forms of the theme, with or
+without modification, are permitted provided that the following conditions
+are met:
+
+* Redistributions of source code must retain the above copyright
+  notice, this list of conditions and the following disclaimer.
+
+* Redistributions in binary form must reproduce the above
+  copyright notice, this list of conditions and the following
+  disclaimer in the documentation and/or other materials provided
+  with the distribution.
+
+* The names of the contributors may not be used to endorse or
+  promote products derived from this software without specific
+  prior written permission.
+
+We kindly ask you to only use these themes in an unmodified manner just
+for Flask and Flask-related products, not for unrelated projects.  If you
+like the visual style and want to use it for your own projects, please
+consider making some larger changes to the themes (such as changing
+font faces, sizes, colors or margins).
+
+THIS THEME IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ARISING IN ANY WAY OUT OF THE USE OF THIS THEME, EVEN IF ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGE.
+"""
+# flasky extensions.  flasky pygments style based on tango style
+from pygments.style import Style
+from pygments.token import Keyword, Name, Comment, String, Error, \
+     Number, Operator, Generic, Whitespace, Punctuation, Other, Literal
+
+
+class FlaskyStyle(Style):
+    background_color = "#f8f8f8"
+    default_style = ""
+
+    styles = {
+        # No corresponding class for the following:
+        #Text:                     "", # class:  ''
+        Whitespace:                "underline #f8f8f8",      # class: 'w'
+        Error:                     "#a40000 border:#ef2929", # class: 'err'
+        Other:                     "#000000",                # class 'x'
+
+        Comment:                   "italic #8f5902", # class: 'c'
+        Comment.Preproc:           "noitalic",       # class: 'cp'
+
+        Keyword:                   "bold #004461",   # class: 'k'
+        Keyword.Constant:          "bold #004461",   # class: 'kc'
+        Keyword.Declaration:       "bold #004461",   # class: 'kd'
+        Keyword.Namespace:         "bold #004461",   # class: 'kn'
+        Keyword.Pseudo:            "bold #004461",   # class: 'kp'
+        Keyword.Reserved:          "bold #004461",   # class: 'kr'
+        Keyword.Type:              "bold #004461",   # class: 'kt'
+
+        Operator:                  "#582800",   # class: 'o'
+        Operator.Word:             "bold #004461",   # class: 'ow' - like keywords
+
+        Punctuation:               "bold #000000",   # class: 'p'
+
+        # because special names such as Name.Class, Name.Function, etc.
+        # are not recognized as such later in the parsing, we choose them
+        # to look the same as ordinary variables.
+        Name:                      "#000000",        # class: 'n'
+        Name.Attribute:            "#c4a000",        # class: 'na' - to be revised
+        Name.Builtin:              "#004461",        # class: 'nb'
+        Name.Builtin.Pseudo:       "#3465a4",        # class: 'bp'
+        Name.Class:                "#000000",        # class: 'nc' - to be revised
+        Name.Constant:             "#000000",        # class: 'no' - to be revised
+        Name.Decorator:            "#888",           # class: 'nd' - to be revised
+        Name.Entity:               "#ce5c00",        # class: 'ni'
+        Name.Exception:            "bold #cc0000",   # class: 'ne'
+        Name.Function:             "#000000",        # class: 'nf'
+        Name.Property:             "#000000",        # class: 'py'
+        Name.Label:                "#f57900",        # class: 'nl'
+        Name.Namespace:            "#000000",        # class: 'nn' - to be revised
+        Name.Other:                "#000000",        # class: 'nx'
+        Name.Tag:                  "bold #004461",   # class: 'nt' - like a keyword
+        Name.Variable:             "#000000",        # class: 'nv' - to be revised
+        Name.Variable.Class:       "#000000",        # class: 'vc' - to be revised
+        Name.Variable.Global:      "#000000",        # class: 'vg' - to be revised
+        Name.Variable.Instance:    "#000000",        # class: 'vi' - to be revised
+
+        Number:                    "#990000",        # class: 'm'
+
+        Literal:                   "#000000",        # class: 'l'
+        Literal.Date:              "#000000",        # class: 'ld'
+
+        String:                    "#4e9a06",        # class: 's'
+        String.Backtick:           "#4e9a06",        # class: 'sb'
+        String.Char:               "#4e9a06",        # class: 'sc'
+        String.Doc:                "italic #8f5902", # class: 'sd' - like a comment
+        String.Double:             "#4e9a06",        # class: 's2'
+        String.Escape:             "#4e9a06",        # class: 'se'
+        String.Heredoc:            "#4e9a06",        # class: 'sh'
+        String.Interpol:           "#4e9a06",        # class: 'si'
+        String.Other:              "#4e9a06",        # class: 'sx'
+        String.Regex:              "#4e9a06",        # class: 'sr'
+        String.Single:             "#4e9a06",        # class: 's1'
+        String.Symbol:             "#4e9a06",        # class: 'ss'
+
+        Generic:                   "#000000",        # class: 'g'
+        Generic.Deleted:           "#a40000",        # class: 'gd'
+        Generic.Emph:              "italic #000000", # class: 'ge'
+        Generic.Error:             "#ef2929",        # class: 'gr'
+        Generic.Heading:           "bold #000080",   # class: 'gh'
+        Generic.Inserted:          "#00A000",        # class: 'gi'
+        Generic.Output:            "#888",           # class: 'go'
+        Generic.Prompt:            "#745334",        # class: 'gp'
+        Generic.Strong:            "bold #000000",   # class: 'gs'
+        Generic.Subheading:        "bold #800080",   # class: 'gu'
+        Generic.Traceback:         "bold #a40000",   # class: 'gt'
+    }

docs/conf.py

@@ -0,0 +1,279 @@
+# -*- coding: utf-8 -*-
+#
+# Jedi documentation build configuration file, created by
+# sphinx-quickstart on Wed Dec 26 00:11:34 2012.
+#
+# This file is execfile()d with the current directory set to its containing dir.
+#
+# Note that not all possible configuration values are present in this
+# autogenerated file.
+#
+# All configuration values have a default; values that are commented out
+# serve to show the default.
+
+import sys, os, datetime
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+sys.path.insert(0, os.path.abspath('..'))
+sys.path.insert(0, os.path.abspath('../jedi'))
+sys.path.append(os.path.abspath('_themes'))
+
+# -- General configuration -----------------------------------------------------
+
+# If your documentation needs a minimal Sphinx version, state it here.
+#needs_sphinx = '1.0'
+
+# Add any Sphinx extension module names here, as strings. They can be extensions
+# coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
+extensions = ['sphinx.ext.autodoc', 'sphinx.ext.viewcode', 'sphinx.ext.todo',
+              'sphinx.ext.intersphinx', 'sphinx.ext.inheritance_diagram']
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# The suffix of source filenames.
+source_suffix = '.rst'
+
+# The encoding of source files.
+source_encoding = 'utf-8'
+
+# The master toctree document.
+master_doc = 'index'
+
+# General information about the project.
+project = u'Jedi'
+copyright = u'2012 - {today.year}, Jedi contributors'.format(today=datetime.date.today())
+
+import jedi
+_path = os.path.dirname(os.path.abspath(__file__))
+_version_strs = [str(x) for x in jedi.__version__]
+
+# The version info for the project you're documenting, acts as replacement for
+# |version| and |release|, also used in various other places throughout the
+# built documents.
+#
+# The short X.Y version.
+
+version = '.'.join(_version_strs[:2])
+# The full version, including alpha/beta/rc tags.
+release = '.'.join(_version_strs)
+
+# The language for content autogenerated by Sphinx. Refer to documentation
+# for a list of supported languages.
+#language = None
+
+# There are two options for replacing |today|: either, you set today to some
+# non-false value, then it is used:
+#today = ''
+# Else, today_fmt is used as the format for a strftime call.
+#today_fmt = '%B %d, %Y'
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+exclude_patterns = []
+
+# The reST default role (used for this markup: `text`) to use for all documents.
+#default_role = None
+
+# If true, '()' will be appended to :func: etc. cross-reference text.
+#add_function_parentheses = True
+
+# If true, the current module name will be prepended to all description
+# unit titles (such as .. function::).
+#add_module_names = True
+
+# If true, sectionauthor and moduleauthor directives will be shown in the
+# output. They are ignored by default.
+#show_authors = False
+
+# The name of the Pygments (syntax highlighting) style to use.
+pygments_style = 'sphinx'
+
+# A list of ignored prefixes for module index sorting.
+#modindex_common_prefix = []
+
+
+# -- Options for HTML output ---------------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+html_theme = 'flask'
+
+# Theme options are theme-specific and customize the look and feel of a theme
+# further.  For a list of options available for each theme, see the
+# documentation.
+#html_theme_options = {}
+
+# Add any paths that contain custom themes here, relative to this directory.
+html_theme_path = ['_themes']
+
+# The name for this set of Sphinx documents.  If None, it defaults to
+# "<project> v<release> documentation".
+#html_title = None
+
+# A shorter title for the navigation bar.  Default is the same as html_title.
+#html_short_title = None
+
+# The name of an image file (relative to this directory) to place at the top
+# of the sidebar.
+#html_logo = None
+
+# The name of an image file (within the static path) to use as favicon of the
+# docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
+# pixels large.
+#html_favicon = None
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ['_static']
+
+# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
+# using the given strftime format.
+#html_last_updated_fmt = '%b %d, %Y'
+
+# If true, SmartyPants will be used to convert quotes and dashes to
+# typographically correct entities.
+#html_use_smartypants = True
+
+# Custom sidebar templates, maps document names to template names.
+html_sidebars = {
+    '**': [
+        'sidebarlogo.html',
+        'localtoc.html',
+#        'relations.html',
+        'ghbuttons.html',
+#        'sourcelink.html',
+#        'searchbox.html'
+    ]
+}
+
+# Additional templates that should be rendered to pages, maps page names to
+# template names.
+#html_additional_pages = {}
+
+# If false, no module index is generated.
+#html_domain_indices = True
+
+# If false, no index is generated.
+#html_use_index = True
+
+# If true, the index is split into individual pages for each letter.
+#html_split_index = False
+
+# If true, links to the reST sources are added to the pages.
+#html_show_sourcelink = True
+
+# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
+#html_show_sphinx = True
+
+# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
+#html_show_copyright = True
+
+# If true, an OpenSearch description file will be output, and all pages will
+# contain a <link> tag referring to it.  The value of this option must be the
+# base URL from which the finished HTML is served.
+#html_use_opensearch = ''
+
+# This is the file name suffix for HTML files (e.g. ".xhtml").
+#html_file_suffix = None
+
+# Output file base name for HTML help builder.
+htmlhelp_basename = 'Jedidoc'
+
+#html_style = 'default.css'  # Force usage of default template on RTD
+
+
+# -- Options for LaTeX output --------------------------------------------------
+
+latex_elements = {
+# The paper size ('letterpaper' or 'a4paper').
+#'papersize': 'letterpaper',
+
+# The font size ('10pt', '11pt' or '12pt').
+#'pointsize': '10pt',
+
+# Additional stuff for the LaTeX preamble.
+#'preamble': '',
+}
+
+# Grouping the document tree into LaTeX files. List of tuples
+# (source start file, target name, title, author, documentclass [howto/manual]).
+latex_documents = [
+  ('index', 'Jedi.tex', u'Jedi Documentation',
+   u'Jedi contributors', 'manual'),
+]
+
+# The name of an image file (relative to this directory) to place at the top of
+# the title page.
+#latex_logo = None
+
+# For "manual" documents, if this is true, then toplevel headings are parts,
+# not chapters.
+#latex_use_parts = False
+
+# If true, show page references after internal links.
+#latex_show_pagerefs = False
+
+# If true, show URL addresses after external links.
+#latex_show_urls = False
+
+# Documents to append as an appendix to all manuals.
+#latex_appendices = []
+
+# If false, no module index is generated.
+#latex_domain_indices = True
+
+
+# -- Options for manual page output --------------------------------------------
+
+# One entry per manual page. List of tuples
+# (source start file, name, description, authors, manual section).
+man_pages = [
+    ('index', 'jedi', u'Jedi Documentation',
+     [u'Jedi contributors'], 1)
+]
+
+# If true, show URL addresses after external links.
+#man_show_urls = False
+
+
+# -- Options for Texinfo output ------------------------------------------------
+
+# Grouping the document tree into Texinfo files. List of tuples
+# (source start file, target name, title, author,
+#  dir menu entry, description, category)
+texinfo_documents = [
+  ('index', 'Jedi', u'Jedi Documentation',
+   u'Jedi contributors', 'Jedi', 'Awesome Python autocompletion library.',
+   'Miscellaneous'),
+]
+
+# Documents to append as an appendix to all manuals.
+#texinfo_appendices = []
+
+# If false, no module index is generated.
+#texinfo_domain_indices = True
+
+# How to display URL addresses: 'footnote', 'no', or 'inline'.
+#texinfo_show_urls = 'footnote'
+
+# -- Options for todo module ---------------------------------------------------
+
+todo_include_todos = False
+
+# -- Options for autodoc module ------------------------------------------------
+
+autoclass_content = 'both'
+autodoc_member_order = 'bysource'
+autodoc_default_flags = []
+#autodoc_default_flags = ['members', 'undoc-members']
+
+
+# -- Options for intersphinx module --------------------------------------------
+
+intersphinx_mapping = {
+    'http://docs.python.org/': None,
+}

docs/docs/development.rst

@@ -0,0 +1,239 @@
+.. include:: ../global.rst
+
+Jedi Development
+================
+
+.. currentmodule:: jedi
+
+.. note:: This documentation is for Jedi developers who want to improve Jedi
+    itself, but have no idea how Jedi works. If you want to use Jedi for 
+    your IDE, look at the `plugin api <plugin-api.html>`_.
+
+
+Introduction
+------------
+
+This page tries to address the fundamental demand for documentation of the
+|jedi| interals. Understanding a dynamic language is a complex task. Especially
+because type inference in Python can be a very recursive task. Therefore |jedi|
+couldn't get rid of complexity. I know that **simple is better than complex**,
+but unfortunately it sometimes requires complex solutions to understand complex
+systems.
+
+Since most of the Jedi internals have been written by me (David Halter), this
+introduction will be written mostly by me, because no one else understands to
+the same level how Jedi works. Actually this is also the reason for exactly this
+part of the documentation. To make multiple people able to edit the Jedi core.
+
+In five chapters I'm trying to describe the internals of |jedi|:
+
+- :ref:`The Jedi Core <core>`
+- :ref:`Core Extensions <core-extensions>`
+- :ref:`Imports & Modules <imports-modules>`
+- :ref:`Caching & Recursions <caching-recursions>`
+- :ref:`Helper modules <dev-helpers>`
+
+.. note:: Testing is not documented here, you'll find that
+   `right here <testing.html>`_.
+
+
+.. _core:
+
+The Jedi Core
+-------------
+
+The core of Jedi consists of three parts:
+
+- :ref:`Parser <parsing>`
+- :ref:`Python code evaluation <evaluate>`
+- :ref:`API <dev-api>`
+
+Most people are probably interested in :ref:`code evaluation <evaluate>`,
+because that's where all the magic happens. I need to introduce the :ref:`parser
+<parsing>` first, because :mod:`evaluate` uses it extensively.
+
+.. _parsing:
+
+Parser (parsing.py)
+~~~~~~~~~~~~~~~~~~~
+
+.. automodule:: parsing
+
+Parser Representation (parser_representation.py)
+++++++++++++++++++++++++++++++++++++++++++++++++
+
+.. automodule:: parsing_representation
+
+Class inheritance diagram:
+
+.. inheritance-diagram::
+   SubModule
+   Class
+   Function
+   Lambda
+   Flow
+   ForFlow
+   Import
+   Statement
+   Param
+   Call
+   Array
+   Name
+   ListComprehension
+   :parts: 1
+
+.. _evaluate:
+
+Evaluation of python code (evaluate.py)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. automodule:: evaluate
+
+Evaluation Representation (evaluate_representation.py)
+++++++++++++++++++++++++++++++++++++++++++++++++++++++
+
+.. automodule:: evaluate_representation
+
+.. inheritance-diagram::
+   Executable
+   Instance
+   InstanceElement
+   Class
+   Function
+   Execution
+   Generator
+   Array
+   :parts: 1
+
+.. _dev-api:
+
+API (api.py and api_classes.py)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The API has been designed to be as easy to use as possible. The API
+documentation can be found `here <plugin-api.html>`_. The API itself contains
+little code that needs to be mentioned here. Generally I'm trying to be
+conservative with the API.  I'd rather not add new API features if they are not
+necessary, because it's much harder to deprecate stuff than to add it later.
+
+
+
+.. _core-extensions:
+
+Core Extensions
+---------------
+
+Core Extensions is a summary of the following topics:
+
+- :ref:`Dynamic Arrays & Function Parameters <dynamic>`
+- :ref:`Fast Parser <fast_parser>`
+- :ref:`Docstrings <docstrings>`
+- :ref:`Refactoring <refactoring>`
+
+These topics are very important to understand what Jedi additionally does, but
+they could be removed from Jedi and Jedi would still work. But slower and
+without some features.
+
+.. _dynamic:
+
+Dynamic Arrays & Function Parameters (dynamic.py)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. automodule:: dynamic
+
+
+.. _fast_parser:
+
+Fast Parser (fast_parser.py)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. automodule:: fast_parser
+
+.. _docstrings:
+
+Docstrings (docstrings.py)
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. automodule:: docstrings
+
+.. _refactoring:
+
+Refactoring (refactoring.py)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. automodule:: refactoring
+
+
+
+.. _imports-modules:
+
+Imports & Modules
+-------------------
+
+
+- :ref:`Modules <modules>`
+- :ref:`Builtin Modules <builtin>`
+- :ref:`Imports <imports>`
+
+.. _modules:
+
+Modules (modules.py)
+~~~~~~~~~~~~~~~~~~~~
+
+.. automodule:: modules
+
+.. _builtin:
+
+Builtin Modules (builtin.py)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. automodule:: builtin
+
+.. _imports:
+
+Imports (imports.py)
+~~~~~~~~~~~~~~~~~~~~
+
+.. automodule:: imports
+
+
+
+.. _caching-recursions:
+
+Caching & Recursions
+----------------------
+
+
+- :ref:`Caching <cache>`
+- :ref:`Recursions <recursion>`
+
+.. _cache:
+
+Caching (cache.py)
+~~~~~~~~~~~~~~~~~~
+
+.. automodule:: cache
+
+.. _recursion:
+
+Recursions (recursion.py)
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. automodule:: recursion
+
+
+
+.. _dev-helpers:
+
+Helper Modules
+---------------
+
+There are some helper modules: `common.py`, `helpers.py`, `debug.py`,
+`keywords.py`, which I won't describe further. Some functions are inheritely
+important and central for Jedi to work, but not important to understand how
+Jedi works.
+
+Python 2/3 compatibility (_compatibility.py)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. automodule:: _compatibility

docs/docs/features.rst

@@ -0,0 +1,98 @@
+.. include:: ../global.rst
+
+Features and Caveats
+====================
+
+|jedi| supports many of the widely used Python features:
+
+
+General Features
+----------------
+
+- python 2.6+ and 3.2+ support
+- ignores syntax errors and wrong indentation
+- can deal with complex module / function / class structures
+- virtualenv support
+- can infer function arguments from sphinx and epydoc docstrings (:ref:`type
+  hinting <type-hinting>`)
+
+
+Supported Python Features
+-------------------------
+
+- builtins
+- multiple returns or yields
+- tuple assignments / array indexing / dictionary indexing
+- with-statement / exception handling
+- ``*args`` / ``**kwargs``
+- decorators / lambdas / closures
+- generators / iterators
+- some descriptors: property / staticmethod / classmethod
+- some magic methods: ``__call__``, ``__iter__``, ``__next__``, ``__get__``,
+  ``__getitem__``, ``__init__``
+- ``list.append()``, ``set.add()``, ``list.extend()``, etc.
+- (nested) list comprehensions / ternary expressions
+- relative imports
+- ``getattr()`` / ``__getattr__`` / ``__getattribute__``
+- function annotations (py3k feature, are ignored right now, but being parsed.
+  I don't know what to do with them.)
+- class decorators (py3k feature, are being ignored too, until I find a use
+  case, that doesn't work with |jedi|)
+- simple/usual ``sys.path`` modifications
+- ``isinstance`` checks for if/while/assert
+- namespace packages (includes ``pkgutil`` and ``pkg_resources`` namespaces)
+
+
+Unsupported Features
+--------------------
+
+Not yet implemented:
+
+- manipulations of instances outside the instance variables without using
+  methods
+
+Will probably never be implemented:
+
+- metaclasses (how could an auto-completion ever support this)
+- ``setattr()``, ``__import__()``
+- writing to some dicts: ``globals()``, ``locals()``, ``object.__dict__``
+- evaluating ``if`` / ``while``
+
+
+Caveats
+-------
+
+**Malformed Syntax**
+
+Syntax errors and other strange stuff may lead to undefined behaviour of the
+completion. |jedi| is **NOT** a Python compiler, that tries to correct you. It
+is a tool that wants to help you. But **YOU** have to know Python, not |jedi|.
+
+**Legacy Python 2 Features**
+
+This framework should work for both Python 2/3. However, some things were just
+not as *pythonic* in Python 2 as things should be. To keep things simple, some
+older Python 2 features have been left out:
+
+- Classes: Always Python 3 like, therefore all classes inherit from ``object``.
+- Generators: No ``next()`` method. The ``__next__()`` method is used instead.
+
+**Slow Performance**
+
+Importing ``numpy`` can be quite slow sometimes, as well as loading the
+builtins the first time. If you want to speed things up, you could write import
+hooks in |jedi|, which preload stuff. However, once loaded, this is not a
+problem anymore.  The same is true for huge modules like ``PySide``, ``wx``,
+etc.
+
+**Security**
+
+Security is an important issue for |jedi|. Therefore no Python code is
+executed.  As long as you write pure python, everything is evaluated
+statically. But: If you use builtin modules (``c_builtin``) there is no other
+option than to execute those modules. However: Execute isn't that critical (as
+e.g. in pythoncomplete, which used to execute *every* import!), because it
+means one import and no more.  So basically the only dangerous thing is using
+the import itself. If your ``c_builtin`` uses some strange initializations, it
+might be dangerous. But if it does you're screwed anyways, because eventualy
+you're going to execute your code, which executes the import.

docs/docs/history.rst

@@ -0,0 +1,31 @@
+.. include:: ../global.rst
+
+A little history
+================
+
+The Star Wars Jedi are awesome. My Jedi software tries to imitate a little bit
+of the precognition the Jedi have. There's even an awesome `scene
+<http://www.youtube.com/watch?v=5BDO3pyavOY>`_ of Monty Python Jedis :-).
+
+But actually the name hasn't so much to do with Star Wars. It's part of my
+second name.
+
+After I explained Guido van Rossum, how some parts of my auto-completion work,
+he said (we drank a beer or two):
+
+    *"Oh, that worries me..."*
+
+When it's finished, I hope he'll like it :-)
+
+I actually started Jedi, because there were no good solutions available for VIM.
+Most auto-completions just didn't work well. The only good solution was PyCharm.
+But I like my good old VIM. Rope was never really intended to be an
+auto-completion (and also I really hate project folders for my Python scripts).
+It's more of a refactoring suite. So I decided to do my own version of a
+completion, which would execute non-dangerous code. But I soon realized, that
+this wouldn't work. So I built an extremely recursive thing which understands
+many of Python's key features.
+
+By the way, I really tried to program it as understandable as possible. But I
+think understanding it might need quite some time, because of its recursive
+nature.

docs/docs/installation.rst

@@ -0,0 +1,81 @@
+.. include:: ../global.rst
+
+Installation and Configuration
+==============================
+
+You can either include |jedi| as a submodule in your text editor plugin (like
+jedi-vim_ does by default), or you can install it systemwide.
+
+.. note:: This just installs the |jedi| library, not the :ref:`editor plugins
+    <editor-plugins>`. For information about how to make it work with your
+    editor, refer to the corresponding documentation.
+
+
+The preferred way
+-----------------
+
+On any system you can install |jedi| directly from the Python package index
+using pip::
+
+    sudo pip install jedi
+
+If you want to install the current development version (master branch)::
+
+    sudo pip install -e git://github.com/davidhalter/jedi.git#egg=jedi
+
+
+System-wide installation via a package manager
+----------------------------------------------
+
+Arch Linux
+~~~~~~~~~~
+
+You can install |jedi| directly from official AUR packages:
+
+- `python-jedi <https://aur.archlinux.org/packages/python-jedi/>`__ (Python 3)
+- `python2-jedi <https://aur.archlinux.org/packages/python2-jedi/>`__ (Python 2)
+
+The specified Python version just refers to the *runtime environment* for
+|jedi|. Use the Python 2 version if you're running vim (or whatever editor you
+use) under Python 2. Otherwise, use the Python 3 version. But whatever version
+you choose, both are able to complete both Python 2 and 3 *code*.
+
+(There is also a packaged version of the vim plugin available: `vim-jedi at AUR
+<https://aur.archlinux.org/packages/vim-jedi/>`__.)
+
+Debian
+~~~~~~
+
+Debian packages are available as `experimental packages
+<http://packages.debian.org/experimental/python-jedi>`__.
+
+Others
+~~~~~~
+
+We are in the discussion of adding |jedi| to the Fedora repositories.
+
+
+Manual installation from a downloaded package
+---------------------------------------------
+
+If you prefer not to use an automated package installer, you can `download
+<https://github.com/davidhalter/jedi/archive/master.zip>`__ a current copy of
+|jedi| and install it manually.
+
+To install it, navigate to the directory containing `setup.py` on your console
+and type::
+
+    sudo python setup.py install
+
+
+Inclusion as a submodule
+------------------------
+
+If you use an editor plugin like jedi-vim_, you can simply include |jedi| as a
+git submodule of the plugin directory. Vim plugin managers like Vundle_ or
+Pathogen_ make it very easy to keep submodules up to date.
+
+
+.. _jedi-vim: https://github.com/davidhalter/jedi-vim
+.. _vundle: https://github.com/gmarik/vundle
+.. _pathogen: https://github.com/tpope/vim-pathogen

docs/docs/plugin-api.rst

@@ -0,0 +1,101 @@
+.. include:: ../global.rst
+
+The Plugin API
+==============
+
+.. currentmodule:: jedi
+
+Note: This documentation is for Plugin developers, who want to improve their
+editors/IDE autocompletion 
+
+If you want to use |jedi|, you first need to
+``import jedi``. You then have direct access to the :class:`.Script`.
+
+
+API documentation
+-----------------
+
+API Interface
+~~~~~~~~~~~~~
+
+.. automodule:: api
+    :members:
+    :undoc-members:
+
+
+
+API Return Classes
+~~~~~~~~~~~~~~~~~~
+
+.. automodule:: api_classes
+    :members:
+    :undoc-members:
+
+Settings Module
+~~~~~~~~~~~~~~~
+
+.. automodule:: settings
+
+Examples
+--------
+
+Completions:
+
+.. sourcecode:: python
+
+   >>> import jedi
+   >>> source = '''import json; json.l'''
+   >>> script = jedi.Script(source, 1, 19, '')
+   >>> script
+   <jedi.api.Script object at 0x2121b10>
+   >>> completions = script.complete()
+   >>> completions
+   [<Completion: load>, <Completion: loads>]
+   >>> completions[1]
+   <Completion: loads>
+   >>> completions[1].complete
+   'oads'
+   >>> completions[1].word
+   'loads'
+
+Definitions / Goto:
+
+.. sourcecode:: python
+
+    >>> import jedi
+    >>> source = '''def my_func():
+    ...     print 'called'
+    ... 
+    ... alias = my_func
+    ... my_list = [1, None, alias]
+    ... inception = my_list[2]
+    ... 
+    ... inception()'''
+    >>> script = jedi.Script(source, 8, 1, '')
+    >>>
+    >>> script.goto()
+    [<Definition inception=my_list[2]>]
+    >>>
+    >>> script.get_definition()
+    [<Definition def my_func>]
+
+Related names:
+
+.. sourcecode:: python
+
+    >>> import jedi
+    >>> source = '''x = 3
+    ... if 1 == 2:
+    ...     x = 4
+    ... else:
+    ...     del x'''
+    >>> script = jedi.Script(source, 5, 8, '')
+    >>> rns = script.related_names()
+    >>> rns
+    [<RelatedName x@3,4>, <RelatedName x@1,0>]
+    >>> rns[0].start_pos
+    (3, 4)
+    >>> rns[0].is_keyword
+    False
+    >>> rns[0].text
+    'x'

docs/docs/recipes.rst

@@ -0,0 +1,44 @@
+.. include:: ../global.rst
+
+Recipes
+=======
+
+Here are some tips on how to use |jedi| efficiently.
+
+
+.. _type-hinting:
+
+Type Hinting
+------------
+
+If |jedi| cannot detect the type of a function argument correctly (due to the
+dynamic nature of Python), you can help it by hinting the type using
+Sphinx-style info field lists or Epydoc docstrings.
+
+**Sphinx style**
+
+http://sphinx-doc.org/domains.html#info-field-lists
+
+::
+
+    def myfunction(node):
+        """Do something with a ``node``.
+
+        :type node: ProgramNode
+
+        """
+        node.| # complete here
+
+**Epydoc**
+
+http://epydoc.sourceforge.net/manual-fields.html
+
+::
+
+    def myfunction(node):
+        """Do something with a ``node``.
+
+        @param node: ProgramNode
+
+        """
+        node.| # complete here

docs/docs/repl.rst

@@ -0,0 +1,18 @@
+.. include:: ../global.rst
+
+Tab completion in the Python Shell
+==================================
+
+There are two different options how you can use Jedi autocompletion in
+your Python interpreter. One with your custom ``$HOME/.pythonrc.py`` file
+and one that uses ``PYTHONSTARTUP``.
+
+Using ``PYTHONSTARTUP``
+-----------------------
+
+.. automodule:: jedi.replstartup
+
+Using a custom ``$HOME/.pythonrc.py``
+-------------------------------------
+
+.. autofunction:: jedi.utils.setup_readline

docs/docs/testing.rst

@@ -0,0 +1,41 @@
+.. include:: ../global.rst
+
+Jedi Testing
+============
+
+The test suite depends on ``tox`` and ``pytest``::
+
+    pip install tox pytest
+
+To run the tests for all supported Python versions::
+
+    tox
+
+If you want to test only a specific Python version (e.g. Python 2.7), it's as
+easy as::
+
+    tox -e py27
+
+Tests are also run automatically on `Travis CI
+<https://travis-ci.org/davidhalter/jedi/>`_.
+
+You want to add a test for |jedi|? Great! We love that. Normally you should
+write your tests as :ref:`Blackbox Tests <blackbox>`. Most tests would
+fit right in there.
+
+.. _blackbox:
+
+Blackbox Tests (run.py)
+~~~~~~~~~~~~~~~~~~~~~~~
+
+.. automodule:: test.run
+
+Regression Tests (test_regression.py)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. automodule:: test.test_regression
+
+Refactoring Tests (refactor.py)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. automodule:: test.refactor

docs/global.rst

@@ -0,0 +1,3 @@
+:orphan:
+
+.. |jedi| replace:: *Jedi*

docs/index.rst

@@ -0,0 +1,68 @@
+.. include global.rst
+
+Jedi - an awesome autocompletion library for Python
+===================================================
+
+Release v\ |release|. (:doc:`Installation <docs/installation>`)
+
+.. automodule:: jedi
+
+Autocompletion can look like this (e.g. VIM plugin):
+
+.. figure:: _screenshots/screenshot_complete.png
+
+
+.. _toc:
+
+Docs
+----
+
+.. toctree::
+   :maxdepth: 1
+
+   docs/installation
+   docs/features
+   docs/repl
+   docs/recipes
+   docs/plugin-api
+   docs/history
+   docs/development
+   docs/testing
+
+
+.. _resources:
+
+Resources
+---------
+
+- `Source Code on Github <https://github.com/davidhalter/jedi>`_
+- `Travis Testing <https://travis-ci.org/davidhalter/jedi>`_
+- `Python Package Index <http://pypi.python.org/pypi/jedi/>`_
+
+
+.. _editor-plugins:
+
+Editor Plugins
+--------------
+
+Vim:
+
+- `jedi-vim <http://github.com/davidhalter/jedi-vim>`_
+- `YouCompleteMe <http://valloric.github.io/YouCompleteMe/>`_
+
+Emacs:
+
+- `Jedi.el <https://github.com/tkf/emacs-jedi>`_
+
+Sublime Text 2/3:
+
+- `SublimeJEDI <https://github.com/srusskih/SublimeJEDI>`_  (ST2 & ST3)
+- `anaconda <https://github.com/DamnWidget/anaconda>`_ (only ST3)
+
+
+.. _other-software:
+
+Other Software Using Jedi
+-------------------------
+
+- `wdb <https://github.com/Kozea/wdb>`_

jedi/__init__.py

@@ -1,16 +1,49 @@
+"""
+Jedi is an autocompletion tool for Python that can be used in IDEs/editors.
+Jedi works. Jedi is fast. It understands all of the basic Python syntax
+elements including many builtin functions.
+
+Additionaly, Jedi suports two different goto functions and has support for
+renaming as well as Pydoc support and some other IDE features.
+
+Jedi uses a very simple API to connect with IDE's. There's a reference
+implementation as a `VIM-Plugin <http://github.com/davidhalter/jedi-vim>`_,
+which uses Jedi's autocompletion.  I encourage you to use Jedi in your IDEs.
+It's really easy. If there are any problems (also with licensing), just contact
+me.
+
+To give you a simple example how you can use the Jedi library, here is an
+example for the autocompletion feature:
+
+>>> import jedi
+>>> source = '''
+... import datetime
+... datetime.da'''
+>>> script = jedi.Script(source, 3, len('datetime.da'), 'example.py')
+>>> script
+<Script: 'example.py'>
+>>> completions = script.completions()
+>>> completions                                         #doctest: +ELLIPSIS
+[<Completion: date>, <Completion: datetime>, ...]
+>>> print(completions[0].complete)
+te
+>>> print(completions[0].name)
+date
+
+As you see Jedi is pretty simple and allows you to concentrate on writing a
+good text editor, while still having very good IDE features for Python.
+"""
+
+__version__ = 0, 7, 0
+
 import sys
 
 # python imports are hell sometimes. Especially the combination of relative
 # imports and circular imports... Just avoid it:
 sys.path.insert(0, __path__[0])
 
-from .api import Script, NotFoundError, set_debug_function
+from .api import Script, Interpreter, NotFoundError, set_debug_function
+from .api import preload_module, defined_names
 from . import settings
 
-from . import api
-
-__doc__ = api.__doc__
-
-del api
-
 sys.path.pop(0)

jedi/__main__.py

@@ -0,0 +1,7 @@
+from sys import argv
+
+if len(argv) == 2 and argv[1] == 'repl':
+    # don't want to use __main__ only for repl yet, maybe we want to use it for
+    # something else. So just use the keyword ``repl`` for now.
+    from os import path
+    print(path.join(path.dirname(path.abspath(__file__)), 'replstartup.py'))

jedi/_compatibility.py

@@ -1,12 +1,60 @@
 """
-This is a compatibility module, to make it possible to use jedi also with older
-python versions.
+To ensure compatibility from Python ``2.6`` - ``3.3``, a module has been
+created. Clearly there is huge need to use conforming syntax. But many changes
+(e.g. ``property``, ``hasattr`` in ``2.5``) can be rewritten in pure python.
 """
 import sys
+import imp
+import os
+try:
+    import importlib
+except:
+    pass
 
 is_py3k = sys.hexversion >= 0x03000000
+is_py33 = sys.hexversion >= 0x03030000
 
-is_py25 = sys.hexversion < 0x02060000
+
+def find_module_py33(string, path=None):
+    loader = importlib.machinery.PathFinder.find_module(string, path)
+
+    if loader is None and path is None:  # Fallback to find builtins
+        loader = importlib.find_loader(string)
+
+    if loader is None:
+        raise ImportError("Couldn't find a loader for {0}".format(string))
+
+    try:
+        is_package = loader.is_package(string)
+        if is_package:
+            module_path = os.path.dirname(loader.path)
+            module_file = None
+        else:
+            module_path = loader.get_filename(string)
+            module_file = open(module_path)
+    except AttributeError:
+        module_path = loader.load_module(string).__name__
+        module_file = None
+
+    return module_file, module_path, is_package
+
+
+def find_module_pre_py33(string, path=None):
+    module_file, module_path, description = imp.find_module(string, path)
+    module_type = description[2]
+    return module_file, module_path, module_type is imp.PKG_DIRECTORY
+
+
+find_module = find_module_py33 if is_py33 else find_module_pre_py33
+find_module.__doc__ = """
+Provides information about a module.
+
+This function isolates the differences in importing libraries introduced with
+python 3.3 on; it gets a module name and optionally a path. It will return a
+tuple containin an open file for the module (if not builtin), the filename
+or the name of the module if it is a builtin one and a boolean indicating
+if the module is contained in a package.
+"""
 
 # next was defined in python 2.6, in python 3 obj.next won't be possible
 # anymore
@@ -26,34 +74,6 @@ except NameError:
             else:
                 return default
 
-# ast module was defined in python 2.6
-try:
-    from ast import literal_eval
-except ImportError:
-    literal_eval = eval
-
-
-# properties in 2.5
-try:
-    property.setter
-except AttributeError:
-    class property(property):
-        def __init__(self, fget, *args, **kwargs):
-            self.__doc__ = fget.__doc__
-            super(property, self).__init__(fget, *args, **kwargs)
-
-        def setter(self, fset):
-            cls_ns = sys._getframe(1).f_locals
-            for k, v in cls_ns.iteritems():
-                if v == self:
-                    propname = k
-                    break
-            cls_ns[propname] = property(self.fget, fset,
-                                        self.fdel, self.__doc__)
-            return cls_ns[propname]
-else:
-    property = property
-
 # unicode function
 try:
     unicode = unicode
@@ -77,6 +97,25 @@ else:
     eval(compile("""def exec_function(source, global_map):
                         exec source in global_map """, 'blub', 'exec'))
 
+# re-raise function
+if is_py3k:
+    def reraise(exception, traceback):
+        raise exception.with_traceback(traceback)
+else:
+    eval(compile("""
+def reraise(exception, traceback):
+    raise exception, None, traceback
+""", 'blub', 'exec'))
+
+reraise.__doc__ = """
+Re-raise `exception` with a `traceback` object.
+
+Usage::
+
+    reraise(Exception, sys.exc_info()[2])
+
+"""
+
 # StringIO (Python 2.5 has no io module), so use io only for py3k
 try:
     from StringIO import StringIO
@@ -105,13 +144,6 @@ class Python3Method(object):
         else:
             return lambda *args, **kwargs: self.func(obj, *args, **kwargs)
 
-try:
-    # the python3 way
-    from functools import reduce
-except ImportError:
-    reduce = reduce
-
-
 def use_metaclass(meta, *bases):
     """ Create a class with a metaclass. """
     if not bases:
@@ -119,51 +151,30 @@ def use_metaclass(meta, *bases):
     return meta("HackClass", bases, {})
 
 try:
-    from inspect import cleandoc
+    from functools import reduce  # Python 3
 except ImportError:
-    # python 2.5 doesn't have this method
-    import string
+    reduce = reduce
 
-    def cleandoc(doc):
-        """Clean up indentation from docstrings.
+try:
+    encoding = sys.stdout.encoding
+    if encoding is None:
+        encoding = 'utf-8'
+except AttributeError:
+    encoding = 'ascii'
 
-        Any whitespace that can be uniformly removed from the second line
-        onwards is removed."""
-        try:
-            lines = string.split(string.expandtabs(doc), '\n')
-        except UnicodeError:
-            return None
-        else:
-            # Find minimum indentation of any non-blank lines after first line.
-            margin = sys.maxint
-            for line in lines[1:]:
-                content = len(string.lstrip(line))
-                if content:
-                    indent = len(line) - content
-                    margin = min(margin, indent)
-            # Remove indentation.
-            if lines:
-                lines[0] = lines[0].lstrip()
-            if margin < sys.maxint:
-                for i in range(1, len(lines)):
-                    lines[i] = lines[i][margin:]
-            # Remove any trailing or leading blank lines.
-            while lines and not lines[-1]:
-                lines.pop()
-            while lines and not lines[0]:
-                lines.pop(0)
-            return string.join(lines, '\n')
+def u(string):
+    """Cast to unicode DAMMIT!
+    Written because Python2 repr always implicitly casts to a string, so we
+    have to cast back to a unicode (and we now that we always deal with valid
+    unicode, because we check that in the beginning).
+    """
+    if is_py3k:
+        return str(string)
+    elif not isinstance(string, unicode):
+        return unicode(str(string), 'UTF-8')
+    return string
 
-if is_py25:
-    # adds the `itertools.chain.from_iterable` constructor
-    import itertools
-
-    class chain(itertools.chain):
-        @staticmethod
-        def from_iterable(iterables):
-            # chain.from_iterable(['ABC', 'DEF']) --> A B C D E F
-            for it in iterables:
-                for element in it:
-                    yield element
-    itertools.chain = chain
-    del chain
+try:
+    import builtins  # module name in python 3
+except ImportError:
+    import __builtin__ as builtins

jedi/api.py

@@ -1,119 +1,171 @@
 """
-Jedi is an autocompletion library for Python. It offers additonal
-services such as goto / get_definition / pydoc support /
-get_in_function_call / related names.
+The API basically only provides one class. You can create a :class:`Script` and
+use its methods.
 
-To give you a simple exmple how you can use the jedi library,
-here is an exmple for the autocompletion feature:
-
->>> import jedi
->>> source = '''import json; json.l'''
->>> script = jedi.Script(source, 1, 19, '')
->>> script
-<jedi.api.Script at 0x7f6d40f3db90>
->>> completions = script.complete()
->>> completions
-[<Completion: load>, <Completion: loads>]
->>> completions[0].complete
-'oad'
->>> completions[0].word
-'load'
-
-As you see Jedi is pretty simple and allows you to concentrate
-writing a good text editor, while still having very good IDE features
-for Python.
+Additionally you can add a debug function with :func:`set_debug_function` and
+catch :exc:`NotFoundError` which is being raised if your completion is not
+possible.
 """
 from __future__ import with_statement
-__all__ = ['Script', 'NotFoundError', 'set_debug_function']
 
 import re
+import os
+import warnings
+from itertools import chain
 
-import parsing
+from jedi import parsing
+from jedi import parsing_representation as pr
+from jedi import debug
+from jedi import settings
+from jedi import helpers
+from jedi import common
+from jedi import cache
+from jedi import modules
+from jedi import interpret
+from jedi._compatibility import next, unicode, builtins
+import keywords
+import evaluate
+import api_classes
+import evaluate_representation as er
 import dynamic
 import imports
-import evaluate
-import modules
-import debug
-import settings
-import keywords
-import helpers
 import builtin
-import api_classes
-
-from _compatibility import next, unicode
 
 
 class NotFoundError(Exception):
-    """ A custom error to avoid catching the wrong exceptions """
-    pass
+    """A custom error to avoid catching the wrong exceptions."""
 
 
 class Script(object):
     """
-    A Script is the base for a completion, goto or whatever call.
+    A Script is the base for completions, goto or whatever you want to do with
+    |jedi|.
 
-    :param source: The source code of the current file
-    :type source: string
-    :param line: The line to complete in.
+    :param source: The source code of the current file, separated by newlines.
+    :type source: str
+    :param line: The line to perform actions on (starting with 1).
     :type line: int
-    :param col: The column to complete in.
+    :param col: The column of the cursor (starting with 0).
     :type col: int
-    :param source_path: The path in the os, the current module is in.
-    :type source_path: string or None
-    :param source_encoding: encoding for decoding `source`, when it
-                            is not a `unicode` object.
-    :type source_encoding: string
+    :param path: The path of the file in the file system, or ``''`` if
+        it hasn't been saved yet.
+    :type path: str or None
+    :param source_encoding: The encoding of ``source``, if it is not a
+        ``unicode`` object (default ``'utf-8'``).
+    :type source_encoding: str
     """
-    def __init__(self, source, line, column, source_path,
-                 source_encoding='utf-8'):
+    def __init__(self, source, line=None, column=None, path=None,
+                 source_encoding='utf-8', source_path=None):
+        if source_path is not None:
+            warnings.warn("Use path instead of source_path.", DeprecationWarning)
+            path = source_path
+
+        lines = source.splitlines()
+        if source and source[-1] == '\n':
+            lines.append('')
+
+        self._line = max(len(lines), 1) if line is None else line
+        self._column = len(lines[-1]) if column is None else column
+
+        api_classes._clear_caches()
         debug.reset_time()
-        try:
-            source = unicode(source, source_encoding, 'replace')
-            # Use 'replace' over 'ignore' to hold code structure.
-        except TypeError:  # `source` is already a unicode object
-            pass
-        self.pos = line, column
-        self.module = modules.ModuleWithCursor(source_path, source=source,
-                                                            position=self.pos)
-        self.source_path = source_path
+        self.source = modules.source_to_unicode(source, source_encoding)
+        self._pos = self._line, self._column
+        self._module = modules.ModuleWithCursor(
+            path, source=self.source, position=self._pos)
+        self._source_path = path
+        self.path = None if path is None else os.path.abspath(path)
         debug.speed('init')
 
     @property
-    def parser(self):
-        """ The lazy parser """
-        return self.module.parser
-
-    def complete(self):
+    def source_path(self):
         """
-        An auto completer for python files.
-
-        :return: list of Completion objects, sorted by name and __ comes last.
-        :rtype: list
+        .. deprecated:: 0.7.0
+           Use :attr:`.path` instead.
+        .. todo:: Remove!
         """
-        def follow_imports_if_possible(name):
-            # TODO remove this, or move to another place (not used)
-            par = name.parent
-            if isinstance(par, parsing.Import) and not \
-                        isinstance(self.parser.user_stmt, parsing.Import):
-                new = imports.ImportPath(par).follow(is_goto=True)
-                # Only remove the old entry if a new one has been found.
-                #print par, new, par.parent
-                if new:
-                    try:
-                        return new
-                    except AttributeError:  # .name undefined
-                        pass
-            return [name]
+        warnings.warn("Use path instead of source_path.", DeprecationWarning)
+        return self.path
 
-        path = self.module.get_path_until_cursor()
-        path, dot, like = self._get_completion_parts(path)
+    def __repr__(self):
+        return '<%s: %s>' % (self.__class__.__name__, repr(self._source_path))
 
+    @property
+    def _parser(self):
+        """ lazy parser."""
+        return self._module.parser
+
+    @api_classes._clear_caches_after_call
+    def completions(self):
+        """
+        Return :class:`api_classes.Completion` objects. Those objects contain
+        information about the completions, more than just names.
+
+        :return: Completion objects, sorted by name and __ comes last.
+        :rtype: list of :class:`api_classes.Completion`
+        """
+        def get_completions(user_stmt, bs):
+            if isinstance(user_stmt, pr.Import):
+                context = self._module.get_context()
+                next(context)  # skip the path
+                if next(context) == 'from':
+                    # completion is just "import" if before stands from ..
+                    return ((k, bs) for k in keywords.keyword_names('import'))
+            return self._simple_complete(path, like)
+
+        debug.speed('completions start')
+        path = self._module.get_path_until_cursor()
+        if re.search('^\.|\.\.$', path):
+            return []
+        path, dot, like = self._get_completion_parts()
+
+        user_stmt = self._user_stmt(True)
+        bs = builtin.Builtin.scope
+        completions = get_completions(user_stmt, bs)
+
+        if not dot:  # named params have no dots
+            for call_def in self.call_signatures():
+                if not call_def.module.is_builtin():
+                    for p in call_def.params:
+                        completions.append((p.get_name(), p))
+
+            if not path and not isinstance(user_stmt, pr.Import):
+                # add keywords
+                completions += ((k, bs) for k in keywords.keyword_names(
+                    all=True))
+
+        needs_dot = not dot and path
+
+        comps = []
+        comp_dct = {}
+        for c, s in set(completions):
+            n = c.names[-1]
+            if settings.case_insensitive_completion \
+                    and n.lower().startswith(like.lower()) \
+                    or n.startswith(like):
+                if not evaluate.filter_private_variable(s,
+                        user_stmt or self._parser.user_scope, n):
+                    new = api_classes.Completion(c, needs_dot, len(like), s)
+                    k = (new.name, new.complete)  # key
+                    if k in comp_dct and settings.no_completion_duplicates:
+                        comp_dct[k]._same_name_completions.append(new)
+                    else:
+                        comp_dct[k] = new
+                        comps.append(new)
+
+        debug.speed('completions end')
+
+        return sorted(comps, key=lambda x: (x.name.startswith('__'),
+                                            x.name.startswith('_'),
+                                            x.name.lower()))
+
+    def _simple_complete(self, path, like):
         try:
             scopes = list(self._prepare_goto(path, True))
         except NotFoundError:
             scopes = []
-            scope_generator = evaluate.get_names_for_scope(
-                                            self.parser.user_scope, self.pos)
+            scope_generator = evaluate.get_names_of_scope(
+                self._parser.user_scope, self._pos)
             completions = []
             for scope, name_list in scope_generator:
                 for c in name_list:
@@ -122,68 +174,55 @@ class Script(object):
             completions = []
             debug.dbg('possible scopes', scopes)
             for s in scopes:
-                if s.isinstance(evaluate.Function):
+                if s.isinstance(er.Function):
                     names = s.get_magic_method_names()
                 else:
                     if isinstance(s, imports.ImportPath):
-                        if like == 'import':
-                            l = self.module.get_line(self.pos[0])[:self.pos[1]]
-                            if not l.endswith('import import'):
+                        under = like + self._module.get_path_after_cursor()
+                        if under == 'import':
+                            current_line = self._module.get_position_line()
+                            if not current_line.endswith('import import'):
                                 continue
+                        a = s.import_stmt.alias
+                        if a and a.start_pos <= self._pos <= a.end_pos:
+                            continue
                         names = s.get_defined_names(on_import_stmt=True)
                     else:
                         names = s.get_defined_names()
 
                 for c in names:
                     completions.append((c, s))
+        return completions
 
-        if not dot:  # named_params have no dots
-            call_def = self.get_in_function_call()
-            if call_def:
-                if not call_def.module.is_builtin():
-                    for p in call_def.params:
-                        completions.append((p.get_name(), p))
+    def _user_stmt(self, is_completion=False):
+        user_stmt = self._parser.user_stmt
+        debug.speed('parsed')
 
-            # Do the completion if there is no path before and no import stmt.
-            if (not scopes or not isinstance(scopes[0], imports.ImportPath)) \
-                        and not path:
-                # add keywords
-                bs = builtin.Builtin.scope
-                completions += ((k, bs) for k in keywords.get_keywords(
-                                                                    all=True))
+        if is_completion and not user_stmt:
+            # for statements like `from x import ` (cursor not in statement)
+            pos = next(self._module.get_context(yield_positions=True))
+            last_stmt = pos and self._parser.module.get_statement_for_position(
+                                pos, include_imports=True)
+            if isinstance(last_stmt, pr.Import):
+                user_stmt = last_stmt
+        return user_stmt
 
-        needs_dot = not dot and path
+    def _prepare_goto(self, goto_path, is_completion=False):
+        """
+        Base for completions/goto. Basically it returns the resolved scopes
+        under cursor.
+        """
+        debug.dbg('start: %s in %s' % (goto_path, self._parser.user_scope))
 
-        comps = []
-        for c, s in set(completions):
-            n = c.names[-1]
-            if settings.case_insensitive_completion \
-                    and n.lower().startswith(like.lower()) \
-                    or n.startswith(like):
-                if not evaluate.filter_private_variable(s,
-                                                    self.parser.user_stmt, n):
-                    new = api_classes.Completion(c, needs_dot,
-                                                    len(like), s)
-                    comps.append(new)
-
-
-        return sorted(comps, key=lambda x: (x.word.startswith('__'),
-                                            x.word.lower()))
-
-    def _prepare_goto(self, goto_path, is_like_search=False):
-        """ Base for complete, goto and get_definition. Basically it returns
-        the resolved scopes under cursor. """
-        debug.dbg('start: %s in %s' % (goto_path, self.parser.scope))
-
-        user_stmt = self.parser.user_stmt
+        user_stmt = self._user_stmt(is_completion)
         if not user_stmt and len(goto_path.split('\n')) > 1:
             # If the user_stmt is not defined and the goto_path is multi line,
             # something's strange. Most probably the backwards tokenizer
             # matched to much.
             return []
 
-        if isinstance(user_stmt, parsing.Import):
-            scopes = [self._get_on_import_stmt(is_like_search)[0]]
+        if isinstance(user_stmt, pr.Import):
+            scopes = [self._get_on_import_stmt(user_stmt, is_completion)[0]]
         else:
             # just parse one statement, take it and evaluate it
             stmt = self._get_under_cursor_stmt(goto_path)
@@ -191,28 +230,90 @@ class Script(object):
         return scopes
 
     def _get_under_cursor_stmt(self, cursor_txt):
-        r = parsing.PyFuzzyParser(cursor_txt, self.source_path, no_docstr=True)
+        offset = self._line - 1, self._column
+        r = parsing.Parser(cursor_txt, no_docstr=True, offset=offset)
         try:
             stmt = r.module.statements[0]
         except IndexError:
             raise NotFoundError()
-        stmt.start_pos = self.pos
-        stmt.parent = self.parser.user_scope
+        stmt.parent = self._parser.user_scope
         return stmt
 
+    def complete(self):
+        """
+        .. deprecated:: 0.6.0
+           Use :attr:`.completions` instead.
+        .. todo:: Remove!
+        """
+        warnings.warn("Use completions instead.", DeprecationWarning)
+        return self.completions()
+
+    def goto(self):
+        """
+        .. deprecated:: 0.6.0
+           Use :attr:`.goto_assignments` instead.
+        .. todo:: Remove!
+        """
+        warnings.warn("Use goto_assignments instead.", DeprecationWarning)
+        return self.goto_assignments()
+
+    def definition(self):
+        """
+        .. deprecated:: 0.6.0
+           Use :attr:`.goto_definitions` instead.
+        .. todo:: Remove!
+        """
+        warnings.warn("Use goto_definitions instead.", DeprecationWarning)
+        return self.goto_definitions()
+
     def get_definition(self):
         """
-        Returns the definitions of a the path under the cursor. This is
-        not a goto function! This follows complicated paths and returns the
-        end, not the first definition.
-        The big difference of goto and get_definition is that goto doesn't
-        follow imports and statements.
-        Multiple objects may be returned, because Python itself is a dynamic
-        language, which means depending on an option you can have two different
-        versions of a function.
+        .. deprecated:: 0.5.0
+           Use :attr:`.goto_definitions` instead.
+        .. todo:: Remove!
+        """
+        warnings.warn("Use goto_definitions instead.", DeprecationWarning)
+        return self.goto_definitions()
 
-        :return: list of Definition objects, which are basically scopes.
-        :rtype: list
+    def related_names(self):
+        """
+        .. deprecated:: 0.6.0
+           Use :attr:`.usages` instead.
+        .. todo:: Remove!
+        """
+        warnings.warn("Use usages instead.", DeprecationWarning)
+        return self.usages()
+
+    def get_in_function_call(self):
+        """
+        .. deprecated:: 0.6.0
+           Use :attr:`.call_signatures` instead.
+        .. todo:: Remove!
+        """
+        return self.function_definition()
+
+    def function_definition(self):
+        """
+        .. deprecated:: 0.6.0
+           Use :attr:`.call_signatures` instead.
+        .. todo:: Remove!
+        """
+        warnings.warn("Use line instead.", DeprecationWarning)
+        sig = self.call_signatures()
+        return sig[0] if sig else None
+
+    @api_classes._clear_caches_after_call
+    def goto_definitions(self):
+        """
+        Return the definitions of a the path under the cursor.  goto function!
+        This follows complicated paths and returns the end, not the first
+        definition. The big difference between :meth:`goto_assignments` and
+        :meth:`goto_definitions` is that :meth:`goto_assignments` doesn't
+        follow imports and statements. Multiple objects may be returned,
+        because Python itself is a dynamic language, which means depending on
+        an option you can have two different versions of a function.
+
+        :rtype: list of :class:`api_classes.Definition`
         """
         def resolve_import_paths(scopes):
             for s in scopes.copy():
@@ -221,43 +322,70 @@ class Script(object):
                     scopes.update(resolve_import_paths(set(s.follow())))
             return scopes
 
-        goto_path = self.module.get_path_under_cursor()
+        goto_path = self._module.get_path_under_cursor()
 
-        context = self.module.get_context()
+        context = self._module.get_context()
+        scopes = set()
+        lower_priority_operators = ('()', '(', ',')
+        """Operators that could hide callee."""
         if next(context) in ('class', 'def'):
-            scopes = set([self.module.parser.user_scope])
+            scopes = set([self._module.parser.user_scope])
         elif not goto_path:
-            op = self.module.get_operator_under_cursor()
-            scopes = set([keywords.get_operator(op, self.pos)] if op else [])
-        else:
-            scopes = set(self._prepare_goto(goto_path))
+            op = self._module.get_operator_under_cursor()
+            if op and op not in lower_priority_operators:
+                scopes = set([keywords.get_operator(op, self._pos)])
+
+        # Fetch definition of callee
+        if not goto_path:
+            (call, _) = self._func_call_and_param_index()
+            if call is not None:
+                while call.next is not None:
+                    call = call.next
+                # reset cursor position:
+                (row, col) = call.name.end_pos
+                _pos = (row, max(col - 1, 0))
+                self._module = modules.ModuleWithCursor(
+                    self._source_path,
+                    source=self.source,
+                    position=_pos)
+                # then try to find the path again
+                goto_path = self._module.get_path_under_cursor()
+
+        if not scopes:
+            if goto_path:
+                scopes = set(self._prepare_goto(goto_path))
+            elif op in lower_priority_operators:
+                scopes = set([keywords.get_operator(op, self._pos)])
 
         scopes = resolve_import_paths(scopes)
 
         # add keywords
-        scopes |= keywords.get_keywords(string=goto_path, pos=self.pos)
+        scopes |= keywords.keywords(string=goto_path, pos=self._pos)
 
         d = set([api_classes.Definition(s) for s in scopes
-                    if not isinstance(s, imports.ImportPath._GlobalNamespace)])
-        return sorted(d, key=lambda x: (x.module_path, x.start_pos))
+                 if not isinstance(s, imports.ImportPath._GlobalNamespace)])
+        return self._sorted_defs(d)
 
-    def goto(self):
+    @api_classes._clear_caches_after_call
+    def goto_assignments(self):
         """
-        Returns the first definition found by goto. This means: It doesn't
-        follow imports and statements.
-        Multiple objects may be returned, because Python itself is a dynamic
-        language, which means depending on an option you can have two different
-        versions of a function.
+        Return the first definition found. Imports and statements aren't
+        followed. Multiple objects may be returned, because Python itself is a
+        dynamic language, which means depending on an option you can have two
+        different versions of a function.
 
-        :return: list of Definition objects, which are basically scopes.
+        :rtype: list of :class:`api_classes.Definition`
         """
-        d = [api_classes.Definition(d) for d in set(self._goto()[0])]
-        return sorted(d, key=lambda x: (x.module_path, x.start_pos))
+        results, _ = self._goto()
+        d = [api_classes.Definition(d) for d in set(results)
+             if not isinstance(d, imports.ImportPath._GlobalNamespace)]
+        return self._sorted_defs(d)
 
     def _goto(self, add_import_name=False):
         """
-        Used for goto and related_names.
-        :param add_import_name: TODO add description
+        Used for goto_assignments and usages.
+
+        :param add_import_name: Add the the name (if import) to the result.
         """
         def follow_inexistent_imports(defs):
             """ Imports can be generated, e.g. following
@@ -266,152 +394,136 @@ class Script(object):
             """
             definitions = set(defs)
             for d in defs:
-                if isinstance(d.parent, parsing.Import) \
-                                        and d.start_pos == (0, 0):
+                if isinstance(d.parent, pr.Import) \
+                        and d.start_pos == (0, 0):
                     i = imports.ImportPath(d.parent).follow(is_goto=True)
                     definitions.remove(d)
                     definitions |= follow_inexistent_imports(i)
             return definitions
 
-        goto_path = self.module.get_path_under_cursor()
-        context = self.module.get_context()
+        goto_path = self._module.get_path_under_cursor()
+        context = self._module.get_context()
+        user_stmt = self._user_stmt()
         if next(context) in ('class', 'def'):
-            user_scope = self.parser.user_scope
+            user_scope = self._parser.user_scope
             definitions = set([user_scope.name])
-            search_name = str(user_scope.name)
-        elif isinstance(self.parser.user_stmt, parsing.Import):
-            s, name_part = self._get_on_import_stmt()
+            search_name = unicode(user_scope.name)
+        elif isinstance(user_stmt, pr.Import):
+            s, name_part = self._get_on_import_stmt(user_stmt)
             try:
                 definitions = [s.follow(is_goto=True)[0]]
             except IndexError:
                 definitions = []
-            search_name = str(name_part)
+            search_name = unicode(name_part)
 
             if add_import_name:
-                import_name = self.parser.user_stmt.get_defined_names()
+                import_name = user_stmt.get_defined_names()
                 # imports have only one name
-                if name_part == import_name[0].names[-1]:
+                if not user_stmt.star \
+                        and name_part == import_name[0].names[-1]:
                     definitions.append(import_name[0])
         else:
             stmt = self._get_under_cursor_stmt(goto_path)
             defs, search_name = evaluate.goto(stmt)
             definitions = follow_inexistent_imports(defs)
+            if isinstance(user_stmt, pr.Statement):
+                c = user_stmt.get_commands()
+                if c and not isinstance(c[0], (str, unicode)) and \
+                   c[0].start_pos > self._pos:
+                    # The cursor must be after the start, otherwise the
+                    # statement is just an assignee.
+                    definitions = [user_stmt]
         return definitions, search_name
 
-    def related_names(self, additional_module_paths=[]):
+    @api_classes._clear_caches_after_call
+    def usages(self, additional_module_paths=()):
         """
-        Returns `dynamic.RelatedName` objects, which contain all names, that
-        are defined by the same variable, function, class or import.
-        This function can be used either to show all the usages of a variable
-        or for renaming purposes.
+        Return :class:`api_classes.Usage` objects, which contain all
+        names that point to the definition of the name under the cursor. This
+        is very useful for refactoring (renaming), or to show all usages of a
+        variable.
 
-        TODO implement additional_module_paths
+        .. todo:: Implement additional_module_paths
+
+        :rtype: list of :class:`api_classes.Usage`
         """
-        user_stmt = self.parser.user_stmt
+        temp, settings.dynamic_flow_information = \
+            settings.dynamic_flow_information, False
+        user_stmt = self._user_stmt()
         definitions, search_name = self._goto(add_import_name=True)
-        if isinstance(user_stmt, parsing.Statement) \
-                    and self.pos < user_stmt.get_assignment_calls().start_pos:
-            # the search_name might be before `=`
-            definitions = [v for v in user_stmt.set_vars
-                                                if str(v) == search_name]
-        if not isinstance(user_stmt, parsing.Import):
+        if isinstance(user_stmt, pr.Statement):
+            c = user_stmt.get_commands()[0]
+            if not isinstance(c, unicode) and self._pos < c.start_pos:
+                # the search_name might be before `=`
+                definitions = [v for v in user_stmt.set_vars
+                               if unicode(v.names[-1]) == search_name]
+        if not isinstance(user_stmt, pr.Import):
             # import case is looked at with add_import_name option
-            definitions = dynamic.related_name_add_import_modules(definitions,
-                                                                search_name)
+            definitions = dynamic.usages_add_import_modules(definitions,
+                                                            search_name)
 
         module = set([d.get_parent_until() for d in definitions])
-        module.add(self.parser.module)
-        names = dynamic.related_names(definitions, search_name, module)
+        module.add(self._parser.module)
+        names = dynamic.usages(definitions, search_name, module)
 
         for d in set(definitions):
-            if isinstance(d, parsing.Module):
-                names.append(api_classes.RelatedName(d, d))
+            if isinstance(d, pr.Module):
+                names.append(api_classes.Usage(d, d))
+            elif isinstance(d, er.Instance):
+                # Instances can be ignored, because they are being created by
+                # ``__getattr__``.
+                pass
             else:
-                names.append(api_classes.RelatedName(d.names[0], d))
+                names.append(api_classes.Usage(d.names[-1], d))
 
-        return sorted(set(names), key=lambda x: (x.module_path, x.start_pos),
-                                                                reverse=True)
+        settings.dynamic_flow_information = temp
+        return self._sorted_defs(set(names))
 
-    def get_in_function_call(self):
+    @api_classes._clear_caches_after_call
+    def call_signatures(self):
         """
-        Return the function, that the cursor is in, e.g.:
-        >>> isinstance(| # | <-- cursor is here
+        Return the function object of the call you're currently in.
 
-        This would return the `isinstance` function. In contrary:
-        >>> isinstance()| # | <-- cursor is here
+        E.g. if the cursor is here::
 
-        This would return `None`.
+            abs(# <-- cursor is here
+
+        This would return the ``abs`` function. On the other hand::
+
+            abs()# <-- cursor is here
+
+        This would return ``None``.
+
+        :rtype: :class:`api_classes.CallDef`
         """
-        def check_user_stmt(user_stmt):
-            if user_stmt is None \
-                        or not isinstance(user_stmt, parsing.Statement):
-                return None, 0
-            ass = helpers.fast_parent_copy(user_stmt.get_assignment_calls())
-
-            call, index, stop = helpers.scan_array_for_pos(ass, self.pos)
-            return call, index
-
-        def check_cache():
-            """ Do the parsing with a part parser, therefore reduce ressource
-            costs.
-            TODO this is not working with multi-line docstrings, improve.
-            """
-            if self.source_path is None:
-                return None, 0
-
-            try:
-                timestamp, parser = builtin.CachedModule.cache[
-                                                            self.source_path]
-            except KeyError:
-                return None, 0
-            part_parser = self.module.get_part_parser()
-            user_stmt = part_parser.user_stmt
-            call, index = check_user_stmt(user_stmt)
-            if call:
-                old_stmt = parser.module.get_statement_for_position(self.pos)
-                if old_stmt is None:
-                    return None, 0
-                old_call, old_index = check_user_stmt(old_stmt)
-                if old_call:
-                    # compare repr because that should definitely be the same.
-                    # Otherwise the whole thing is out of sync.
-                    if repr(old_call) == repr(call):
-                        # return the index of the part_parser
-                        return old_call, index
-                return None, 0
-            else:
-                raise NotFoundError()
-
-        debug.speed('func_call start')
-        try:
-            call, index = check_cache()
-        except NotFoundError:
-            return None
-        debug.speed('func_call parsed')
 
+        call, index = self._func_call_and_param_index()
         if call is None:
-            # This is a backup, if the above is not successful.
-            user_stmt = self.parser.user_stmt
-            call, index = check_user_stmt(user_stmt)
-            if call is None:
-                return None
+            return []
 
-        debug.speed('func_call user_stmt')
-        with helpers.scale_speed_settings(settings.scale_get_in_function_call):
-            origins = evaluate.follow_call(call)
+        user_stmt = self._user_stmt()
+        with common.scale_speed_settings(settings.scale_function_definition):
+            _callable = lambda: evaluate.follow_call(call)
+            origins = cache.cache_function_definition(_callable, user_stmt)
         debug.speed('func_call followed')
 
-        if len(origins) == 0:
-            return None
-        # just take entry zero, because we need just one.
-        executable = origins[0]
+        return [api_classes.CallDef(o, index, call) for o in origins
+                if o.isinstance(er.Function, er.Instance, er.Class)]
 
-        return api_classes.CallDef(executable, index, call)
+    def _func_call_and_param_index(self):
+        debug.speed('func_call start')
+        call, index = None, 0
+        if call is None:
+            user_stmt = self._user_stmt()
+            if user_stmt is not None and isinstance(user_stmt, pr.Statement):
+                call, index, _ = helpers.search_function_definition(
+                    user_stmt, self._pos)
+        debug.speed('func_call parsed')
+        return call, index
 
-    def _get_on_import_stmt(self, is_like_search=False):
+    def _get_on_import_stmt(self, user_stmt, is_like_search=False):
         """ Resolve the user statement, if it is an import. Only resolve the
         parts until the user position. """
-        user_stmt = self.parser.user_stmt
         import_names = user_stmt.get_all_import_names()
         kill_count = -1
         cur_name_part = None
@@ -419,31 +531,155 @@ class Script(object):
             if user_stmt.alias == i:
                 continue
             for name_part in i.names:
-                if name_part.end_pos >= self.pos:
+                if name_part.end_pos >= self._pos:
                     if not cur_name_part:
                         cur_name_part = name_part
                     kill_count += 1
 
+
+        context = self._module.get_context()
+        just_from = next(context) == 'from'
+
         i = imports.ImportPath(user_stmt, is_like_search,
-                                kill_count=kill_count, direct_resolve=True)
+                               kill_count=kill_count, direct_resolve=True,
+                               is_just_from=just_from)
         return i, cur_name_part
 
-    def _get_completion_parts(self, path):
+    def _get_completion_parts(self):
         """
         Returns the parts for the completion
         :return: tuple - (path, dot, like)
         """
+        path = self._module.get_path_until_cursor()
         match = re.match(r'^(.*?)(\.|)(\w?[\w\d]*)$', path, flags=re.S)
         return match.groups()
 
-    def __del__(self):
-        evaluate.clear_caches()
+    @staticmethod
+    def _sorted_defs(d):
+        # Note: `or ''` below is required because `module_path` could be
+        #       None and you can't compare None and str in Python 3.
+        return sorted(d, key=lambda x: (x.module_path or '', x.line, x.column))
+
+
+class Interpreter(Script):
+
+    """
+    Jedi API for Python REPLs.
+
+    In addition to completion of simple attribute access, Jedi
+    supports code completion based on static code analysis.
+    Jedi can complete attributes of object which is not initialized
+    yet.
+
+    >>> from os.path import join
+    >>> namespace = locals()
+    >>> script = Interpreter('join().up', [namespace])
+    >>> print(script.completions()[0].name)
+    upper
+
+    """
+
+    def __init__(self, source, namespaces=[], **kwds):
+        """
+        Parse `source` and mixin interpreted Python objects from `namespaces`.
+
+        :type source: str
+        :arg  source: Code to parse.
+        :type namespaces: list of dict
+        :arg  namespaces: a list of namespace dictionaries such as the one
+                          returned by :func:`locals`.
+
+        Other optional arguments are same as the ones for :class:`Script`.
+        If `line` and `column` are None, they are assumed be at the end of
+        `source`.
+        """
+        super(Interpreter, self).__init__(source, **kwds)
+        self.namespaces = namespaces
+
+        # Here we add the namespaces to the current parser.
+        importer = interpret.ObjectImporter(self._parser.user_scope)
+        for ns in namespaces:
+            importer.import_raw_namespace(ns)
+
+    def _simple_complete(self, path, like):
+        user_stmt = self._user_stmt(True)
+        is_simple_path = not path or re.search('^[\w][\w\d.]*$', path)
+        if isinstance(user_stmt, pr.Import) or not is_simple_path:
+            return super(type(self), self)._simple_complete(path, like)
+        else:
+            class NamespaceModule:
+                def __getattr__(_, name):
+                    for n in self.namespaces:
+                        try:
+                            return n[name]
+                        except KeyError:
+                            pass
+                    raise AttributeError()
+
+                def __dir__(_):
+                    return list(set(chain.from_iterable(n.keys()
+                                    for n in self.namespaces)))
+
+            paths = path.split('.') if path else []
+
+            namespaces = (NamespaceModule(), builtins)
+            for p in paths:
+                old, namespaces = namespaces, []
+                for n in old:
+                    try:
+                        namespaces.append(getattr(n, p))
+                    except AttributeError:
+                        pass
+
+            completions = []
+            for n in namespaces:
+                for name in dir(n):
+                    if name.lower().startswith(like.lower()):
+                        scope = self._parser.module
+                        n = pr.Name(self._parser.module, [(name, (0, 0))],
+                                    (0, 0), (0, 0), scope)
+                        completions.append((n, scope))
+            return completions
+
+
+
+
+def defined_names(source, path=None, source_encoding='utf-8'):
+    """
+    Get all definitions in `source` sorted by its position.
+
+    This functions can be used for listing functions, classes and
+    data defined in a file.  This can be useful if you want to list
+    them in "sidebar".  Each element in the returned list also has
+    `defined_names` method which can be used to get sub-definitions
+    (e.g., methods in class).
+
+    :rtype: list of api_classes.Definition
+    """
+    parser = parsing.Parser(
+        modules.source_to_unicode(source, source_encoding),
+        module_path=path,
+    )
+    return api_classes._defined_names(parser.module)
+
+
+def preload_module(*modules):
+    """
+    Preloading modules tells Jedi to load a module now, instead of lazy parsing
+    of modules. Usful for IDEs, to control which modules to load on startup.
+
+    :param modules: different module names, list of string.
+    """
+    for m in modules:
+        s = "import %s as x; x." % m
+        Script(s, 1, len(s), None).completions()
 
 
 def set_debug_function(func_cb=debug.print_to_stdout, warnings=True,
-                                            notices=True, speed=True):
+                       notices=True, speed=True):
     """
-    You can define a callback debug function to get all the debug messages.
+    Define a callback debug function to get all the debug messages.
+
     :param func_cb: The callback function for debug messages, with n params.
     """
     debug.debug_function = func_cb

jedi/api_classes.py

@@ -1,26 +1,67 @@
-""" The classes returned by the api """
+"""
+The :mod:`api_classes` module contains the return classes of the API. These
+classes are the much bigger part of the whole API, because they contain the
+interesting information about completion and goto operations.
+"""
+from __future__ import with_statement
 
-import re
-import os
+import warnings
+import functools
 
-import settings
+from jedi._compatibility import unicode, next
+from jedi import settings
+from jedi import common
+from jedi import parsing_representation as pr
+from jedi import cache
+import keywords
+import recursion
+import dynamic
 import evaluate
 import imports
-import parsing
-import keywords
+import evaluate_representation as er
+
+
+def _clear_caches():
+    """
+    Clear all caches of this and related modules. The only cache that will not
+    be deleted is the module cache.
+    """
+    cache.clear_caches()
+    dynamic.search_param_cache.clear()
+    recursion.ExecutionRecursionDecorator.reset()
+
+    evaluate.follow_statement.reset()
+
+    imports.imports_processed = 0
+
+
+def _clear_caches_after_call(func):
+    """
+    Clear caches just before returning a value.
+    """
+    @functools.wraps(func)
+    def wrapper(*args, **kwds):
+        result = func(*args, **kwds)
+        _clear_caches()
+        return result
+    return wrapper
 
 
 class BaseDefinition(object):
-    _mapping = {'posixpath': 'os.path',
-               'riscospath': 'os.path',
-               'ntpath': 'os.path',
-               'os2emxpath': 'os.path',
-               'macpath': 'os.path',
-               'genericpath': 'os.path',
-               '_io': 'io',
-               '__builtin__': '',
-               'builtins': '',
-               }
+    _mapping = {
+        'posixpath': 'os.path',
+        'riscospath': 'os.path',
+        'ntpath': 'os.path',
+        'os2emxpath': 'os.path',
+        'macpath': 'os.path',
+        'genericpath': 'os.path',
+        'posix': 'os',
+        '_io': 'io',
+        '_functools': 'functools',
+        '_sqlite3': 'sqlite3',
+        '__builtin__': '',
+        'builtins': '',
+    }
 
     _tuple_mapping = dict((tuple(k.split('.')), v) for (k, v) in {
         'argparse._ActionsContainer': 'argparse.ArgumentParser',
@@ -29,83 +70,228 @@ class BaseDefinition(object):
     }.items())
 
     def __init__(self, definition, start_pos):
-        self.start_pos = start_pos
-        self.definition = definition
+        self._start_pos = start_pos
+        self._definition = definition
+        """
+        An instance of :class:`jedi.parsing_representation.Base` subclass.
+        """
         self.is_keyword = isinstance(definition, keywords.Keyword)
 
         # generate a path to the definition
-        self.module_path = str(definition.get_parent_until().path)
+        self._module = definition.get_parent_until()
+        self.module_path = self._module.path
+
+    @property
+    def start_pos(self):
+        """
+        .. deprecated:: 0.7.0
+           Use :attr:`.line` and :attr:`.column` instead.
+        .. todo:: Remove!
+        """
+        warnings.warn("Use line/column instead.", DeprecationWarning)
+        return self._start_pos
 
     @property
     def type(self):
+        """
+        The type of the definition.
+
+        Here is an example of the value of this attribute.  Let's consider
+        the following source.  As what is in ``variable`` is unambiguous
+        to Jedi, :meth:`api.Script.goto_definitions` should return a list of
+        definition for ``sys``, ``f``, ``C`` and ``x``.
+
+        >>> from jedi import Script
+        >>> source = '''
+        ... import keyword
+        ...
+        ... class C:
+        ...     pass
+        ...
+        ... class D:
+        ...     pass
+        ...
+        ... x = D()
+        ...
+        ... def f():
+        ...     pass
+        ...
+        ... variable = keyword or f or C or x'''
+        >>> script = Script(source, len(source.splitlines()), 3, 'example.py')
+        >>> defs = script.goto_definitions()
+
+        Before showing what is in ``defs``, let's sort it by :attr:`line`
+        so that it is easy to relate the result to the source code.
+
+        >>> defs = sorted(defs, key=lambda d: d.line)
+        >>> defs                           # doctest: +NORMALIZE_WHITESPACE
+        [<Definition module keyword>, <Definition class C>,
+         <Definition class D>, <Definition def f>]
+
+        Finally, here is what you can get from :attr:`type`:
+
+        >>> defs[0].type
+        'module'
+        >>> defs[1].type
+        'class'
+        >>> defs[2].type
+        'instance'
+        >>> defs[3].type
+        'function'
+
+        """
         # generate the type
-        stripped = self.definition
-        if isinstance(self.definition, evaluate.InstanceElement):
-            stripped = self.definition.var
-        return type(stripped).__name__
+        stripped = self._definition
+        if isinstance(self._definition, er.InstanceElement):
+            stripped = self._definition.var
+        if isinstance(stripped, pr.Name):
+            stripped = stripped.parent
+        return type(stripped).__name__.lower()
 
     @property
     def path(self):
+        """The module path."""
         path = []
-        if not isinstance(self.definition, keywords.Keyword):
-            par = self.definition
+
+        def insert_nonnone(x):
+            if x:
+                path.insert(0, x)
+
+        if not isinstance(self._definition, keywords.Keyword):
+            par = self._definition
             while par is not None:
-                try:
+                if isinstance(par, pr.Import):
+                    insert_nonnone(par.namespace)
+                    insert_nonnone(par.from_ns)
+                    if par.relative_count == 0:
+                        break
+                with common.ignored(AttributeError):
                     path.insert(0, par.name)
-                except AttributeError:
-                    pass
                 par = par.parent
         return path
 
     @property
     def module_name(self):
-        path = self.module_path
-        sep = os.path.sep
-        p = re.sub(r'^.*?([\w\d]+)(%s__init__)?.py$' % sep, r'\1', path)
-        return p
+        """
+        The module name.
+
+        >>> from jedi import Script
+        >>> source = 'import datetime'
+        >>> script = Script(source, 1, len(source), 'example.py')
+        >>> d = script.goto_definitions()[0]
+        >>> print(d.module_name)                       # doctest: +ELLIPSIS
+        datetime
+        """
+        return str(self._module.name)
 
     def in_builtin_module(self):
-        return not self.module_path.endswith('.py')
+        """Whether this is a builtin module."""
+        return not (self.module_path is None or
+                    self.module_path.endswith('.py'))
 
     @property
     def line_nr(self):
-        return self.start_pos[0]
+        """
+        .. deprecated:: 0.5.0
+           Use :attr:`.line` instead.
+        .. todo:: Remove!
+        """
+        warnings.warn("Use line instead.", DeprecationWarning)
+        return self.line
+
+    @property
+    def line(self):
+        """The line where the definition occurs (starting with 1)."""
+        if self.in_builtin_module():
+            return None
+        return self._start_pos[0]
 
     @property
     def column(self):
-        return self.start_pos[1]
+        """The column where the definition occurs (starting with 0)."""
+        if self.in_builtin_module():
+            return None
+        return self._start_pos[1]
 
     @property
     def doc(self):
-        """ Return a document string for this completion object. """
+        r"""
+        Return a document string for this completion object.
+
+        Example:
+
+        >>> from jedi import Script
+        >>> source = '''\
+        ... def f(a, b=1):
+        ...     "Document for function f."
+        ... '''
+        >>> script = Script(source, 1, len('def f'), 'example.py')
+        >>> d = script.goto_definitions()[0]
+        >>> print(d.doc)
+        f(a, b = 1)
+        <BLANKLINE>
+        Document for function f.
+
+        Notice that useful extra information is added to the actual
+        docstring.  For function, it is call signature.  If you need
+        actual docstring, use :attr:`raw_doc` instead.
+
+        >>> print(d.raw_doc)
+        Document for function f.
+
+        """
         try:
-            return self.definition.doc
+            return self._definition.doc
         except AttributeError:
             return self.raw_doc
 
     @property
     def raw_doc(self):
-        """ Returns the raw docstring `__doc__` for any object """
+        """
+        The raw docstring ``__doc__`` for any object.
+
+        See :attr:`doc` for example.
+        """
         try:
-            return str(self.definition.docstr)
+            return unicode(self._definition.docstr)
         except AttributeError:
             return ''
 
     @property
     def description(self):
-        return str(self.definition)
+        """A textual description of the object."""
+        return unicode(self._definition)
 
     @property
     def full_name(self):
         """
-        Returns the path to a certain class/function, see #61.
+        Dot-separated path of this object.
+
+        It is in the form of ``<module>[.<submodule>[...]][.<object>]``.
+        It is useful when you want to look up Python manual of the
+        object at hand.
+
+        Example:
+
+        >>> from jedi import Script
+        >>> source = '''
+        ... import os
+        ... os.path.join'''
+        >>> script = Script(source, 3, len('os.path.join'), 'example.py')
+        >>> print(script.goto_definitions()[0].full_name)
+        os.path.join
+
+        Notice that it correctly returns ``'os.path.join'`` instead of
+        (for example) ``'posixpath.join'``.
+
         """
-        path = [str(p) for p in self.path]
+        path = [unicode(p) for p in self.path]
         # TODO add further checks, the mapping should only occur on stdlib.
-        try:
+        if not path:
+            return None  # for keywords the path is empty
+
+        with common.ignored(KeyError):
             path[0] = self._mapping[path[0]]
-        except KeyError:
-            pass
         for key, repl in self._tuple_mapping.items():
             if tuple(path[:len(key)]) == key:
                 path = [repl] + path[len(key):]
@@ -117,173 +303,308 @@ class BaseDefinition(object):
 
 
 class Completion(BaseDefinition):
-    """ `Completion` objects are returned from `Script.complete`. Providing
-    some useful functions for IDE's. """
+    """
+    `Completion` objects are returned from :meth:`api.Script.completions`. They
+    provide additional information about a completion.
+    """
     def __init__(self, name, needs_dot, like_name_length, base):
         super(Completion, self).__init__(name.parent, name.start_pos)
 
-        self.name = name
-        self.needs_dot = needs_dot
-        self.like_name_length = like_name_length
-        self.base = base
+        self._name = name
+        self._needs_dot = needs_dot
+        self._like_name_length = like_name_length
+        self._base = base
+
+        # Completion objects with the same Completion name (which means
+        # duplicate items in the completion)
+        self._same_name_completions = []
 
         self._followed_definitions = None
 
-    @property
-    def complete(self):
-        """ Delievers the rest of the word, e.g. completing `isinstance`
-        >>> isinstan
-
-        would return the string 'ce'. It also adds additional stuff, depending
-        on your `settings.py`
-        """
-        dot = '.' if self.needs_dot else ''
+    def _complete(self, like_name):
+        dot = '.' if self._needs_dot else ''
         append = ''
         if settings.add_bracket_after_function \
-                    and self.type == 'Function':
+                and self.type == 'Function':
             append = '('
 
         if settings.add_dot_after_module:
-            if isinstance(self.base, parsing.Module):
+            if isinstance(self._base, pr.Module):
                 append += '.'
-        if isinstance(self.base, parsing.Param):
+        if isinstance(self._base, pr.Param):
             append += '='
-        return dot + self.name.names[-1][self.like_name_length:] + append
+
+        name = self._name.names[-1]
+        if like_name:
+            name = name[self._like_name_length:]
+        return dot + name + append
+
+    @property
+    def complete(self):
+        """
+        Return the rest of the word, e.g. completing ``isinstance``::
+
+            isinstan# <-- Cursor is here
+
+        would return the string 'ce'. It also adds additional stuff, depending
+        on your `settings.py`.
+        """
+        return self._complete(True)
+
+    @property
+    def name(self):
+        """
+        Similar to :meth:`Completion.complete`, but return the whole word, for
+        example::
+
+            isinstan
+
+        would return `isinstance`.
+        """
+        return unicode(self._name.names[-1])
+
+    @property
+    def name_with_symbols(self):
+        """
+        Similar to :meth:`Completion.name`, but like :meth:`Completion.name`
+        returns also the symbols, for example::
+
+            list()
+
+        would return ``.append`` and others (which means it adds a dot).
+        """
+        return self._complete(False)
 
     @property
     def word(self):
-        """ In contrary to `complete` returns the whole word, e.g.
-        >>> isinstan
-
-        would return 'isinstance'.
         """
-        return str(self.name.names[-1])
+        .. deprecated:: 0.6.0
+           Use :attr:`.name` instead.
+        .. todo:: Remove!
+        """
+        warnings.warn("Use name instead.", DeprecationWarning)
+        return self.name
 
     @property
     def description(self):
-        """ Provides a description of the completion object
-        TODO return value is just __repr__ of some objects, improve! """
-        parent = self.name.parent
+        """Provide a description of the completion object."""
+        parent = self._name.parent
         if parent is None:
             return ''
         t = self.type
-        if t == 'Statement' or t == 'Import':
-            desc = self.definition.get_code(False)
+        if t == 'statement' or t == 'import':
+            desc = self._definition.get_code(False)
         else:
-            desc = '.'.join(str(p) for p in self.path)
+            desc = '.'.join(unicode(p) for p in self.path)
 
-        line_nr = '' if self.in_builtin_module else '@%s' % self.line_nr
-        return '%s: %s%s' % (t, desc, line_nr)
+        line = '' if self.in_builtin_module else '@%s' % self.line
+        return '%s: %s%s' % (t, desc, line)
 
     def follow_definition(self):
-        """ Returns you the original definitions. I strongly recommend not
-        using it for your completions, because it might slow down Jedi. If you
-        want to read only a few objects (<=20). I think it might be useful,
-        especially to get the original docstrings.
-        The basic problem of this function is that it follows all results. This
-        means with 1000 completions (e.g. numpy), it's just PITA slow.
+        """
+        Return the original definitions. I strongly recommend not using it for
+        your completions, because it might slow down |jedi|. If you want to
+        read only a few objects (<=20), it might be useful, especially to get
+        the original docstrings. The basic problem of this function is that it
+        follows all results. This means with 1000 completions (e.g.  numpy),
+        it's just PITA-slow.
         """
         if self._followed_definitions is None:
-            if self.definition.isinstance(parsing.Statement):
-                defs = evaluate.follow_statement(self.definition)
-            elif self.definition.isinstance(parsing.Import):
-                defs = imports.strip_imports([self.definition])
+            if self._definition.isinstance(pr.Statement):
+                defs = evaluate.follow_statement(self._definition)
+            elif self._definition.isinstance(pr.Import):
+                defs = imports.strip_imports([self._definition])
             else:
                 return [self]
 
             self._followed_definitions = \
-                            [BaseDefinition(d, d.start_pos) for d in defs]
-            evaluate.clear_caches()
+                [BaseDefinition(d, d.start_pos) for d in defs]
+            _clear_caches()
 
         return self._followed_definitions
 
     def __repr__(self):
-        return '<%s: %s>' % (type(self).__name__, self.name)
+        return '<%s: %s>' % (type(self).__name__, self._name)
 
 
 class Definition(BaseDefinition):
-    """ These are the objects returned by either `Script.goto` or
-    `Script.get_definition`. """
+    """
+    *Definition* objects are returned from :meth:`api.Script.goto_assignments`
+    or :meth:`api.Script.goto_definitions`.
+    """
     def __init__(self, definition):
         super(Definition, self).__init__(definition, definition.start_pos)
 
     @property
-    def description(self):
-        """ A description of the Definition object, which is heavily used in
-        testing. e.g. for `isinstance` it returns 'def isinstance' """
-        d = self.definition
-        if isinstance(d, evaluate.InstanceElement):
+    def name(self):
+        """
+        Name of variable/function/class/module.
+
+        For example, for ``x = None`` it returns ``'x'``.
+
+        :rtype: str or None
+        """
+        d = self._definition
+        if isinstance(d, er.InstanceElement):
             d = d.var
-        if isinstance(d, evaluate.parsing.Name):
+
+        if isinstance(d, pr.Name):
+            return d.names[-1] if d.names else None
+        elif isinstance(d, er.Array):
+            return unicode(d.type)
+        elif isinstance(d, (pr.Class, er.Class, er.Instance,
+                            er.Function, pr.Function)):
+            return unicode(d.name)
+        elif isinstance(d, pr.Module):
+            return self.module_name
+        elif isinstance(d, pr.Import):
+            try:
+                return d.get_defined_names()[0].names[-1]
+            except (AttributeError, IndexError):
+                return None
+        elif isinstance(d, pr.Statement):
+            try:
+                return d.assignment_details[0][1].values[0][0].name.names[-1]
+            except IndexError:
+                return None
+        return None
+
+    @property
+    def description(self):
+        """
+        A description of the :class:`.Definition` object, which is heavily used
+        in testing. e.g. for ``isinstance`` it returns ``def isinstance``.
+
+        Example:
+
+        >>> from jedi import Script
+        >>> source = '''
+        ... def f():
+        ...     pass
+        ...
+        ... class C:
+        ...     pass
+        ...
+        ... variable = f or C'''
+        >>> script = Script(source, column=3)  # line is maximum by default
+        >>> defs = script.goto_definitions()
+        >>> defs = sorted(defs, key=lambda d: d.line)
+        >>> defs
+        [<Definition def f>, <Definition class C>]
+        >>> str(defs[0].description)  # strip literals in python2
+        'def f'
+        >>> str(defs[1].description)
+        'class C'
+
+        """
+        d = self._definition
+        if isinstance(d, er.InstanceElement):
+            d = d.var
+        if isinstance(d, pr.Name):
             d = d.parent
 
-        if isinstance(d, evaluate.Array):
+        if isinstance(d, er.Array):
             d = 'class ' + d.type
-        elif isinstance(d, (parsing.Class, evaluate.Class, evaluate.Instance)):
-            d = 'class ' + str(d.name)
-        elif isinstance(d, (evaluate.Function, evaluate.parsing.Function)):
-            d = 'def ' + str(d.name)
-        elif isinstance(d, evaluate.parsing.Module):
+        elif isinstance(d, (pr.Class, er.Class, er.Instance)):
+            d = 'class ' + unicode(d.name)
+        elif isinstance(d, (er.Function, pr.Function)):
+            d = 'def ' + unicode(d.name)
+        elif isinstance(d, pr.Module):
             # only show module name
             d = 'module %s' % self.module_name
         elif self.is_keyword:
             d = 'keyword %s' % d.name
         else:
-            d = d.get_code().replace('\n', '')
+            code = d.get_code().replace('\n', '')
+            max_len = 20
+            d = (code[:max_len] + '...') if len(code) > max_len + 3 else code
         return d
 
     @property
     def desc_with_module(self):
-        """ In addition to the Definition, it also returns the module. Don't
-        use it yet, its behaviour may change. If you really need it, talk to me
-        TODO add full path. This function is should return a
-        module.class.function path. """
+        """
+        In addition to the definition, also return the module.
+
+        .. warning:: Don't use this function yet, its behaviour may change. If
+            you really need it, talk to me.
+
+        .. todo:: Add full path. This function is should return a
+            `module.class.function` path.
+        """
         if self.module_path.endswith('.py') \
-                    and not isinstance(self.definition, parsing.Module):
-            position = '@%s' % (self.line_nr)
+                and not isinstance(self._definition, pr.Module):
+            position = '@%s' % (self.line)
         else:
             # is a builtin or module
             position = ''
         return "%s:%s%s" % (self.module_name, self.description, position)
 
+    def defined_names(self):
+        """
+        List sub-definitions (e.g., methods in class).
 
-class RelatedName(BaseDefinition):
+        :rtype: list of Definition
+        """
+        d = self._definition
+        if isinstance(d, er.InstanceElement):
+            d = d.var
+        if isinstance(d, pr.Name):
+            d = d.parent
+        return _defined_names(d)
+
+
+def _defined_names(scope):
+    """
+    List sub-definitions (e.g., methods in class).
+
+    :type scope: Scope
+    :rtype: list of Definition
+    """
+    pair = next(evaluate.get_names_of_scope(
+        scope, star_search=False, include_builtin=False), None)
+    names = pair[1] if pair else []
+    return [Definition(d) for d in sorted(names, key=lambda s: s.start_pos)]
+
+
+class Usage(BaseDefinition):
+    """TODO: document this"""
     def __init__(self, name_part, scope):
-        super(RelatedName, self).__init__(scope, name_part.start_pos)
-        self.name_part = name_part
-        self.text = str(name_part)
+        super(Usage, self).__init__(scope, name_part.start_pos)
+        self.text = unicode(name_part)
         self.end_pos = name_part.end_pos
 
     @property
     def description(self):
-        return "%s@%s,%s" % (self.text, self.start_pos[0], self.start_pos[1])
+        return "%s@%s,%s" % (self.text, self.line, self.column)
 
     def __eq__(self, other):
-        return self.start_pos == other.start_pos \
+        return self._start_pos == other._start_pos \
             and self.module_path == other.module_path
 
     def __hash__(self):
-        return hash((self.start_pos, self.module_path))
+        return hash((self._start_pos, self.module_path))
 
 
 class CallDef(object):
-    """ `CallDef` objects is the return value of `Script.get_in_function_call`.
+    """
+    `CallDef` objects is the return value of `Script.function_definition`.
     It knows what functions you are currently in. e.g. `isinstance(` would
-    return the `isinstance` function. without `(` it would return nothing."""
+    return the `isinstance` function. without `(` it would return nothing.
+    """
     def __init__(self, executable, index, call):
-        self.executable = executable
+        self._executable = executable
         self.index = index
-        self.call = call
+        self._call = call
 
     @property
     def params(self):
-        if self.executable.isinstance(evaluate.Function):
-            if isinstance(self.executable, evaluate.InstanceElement):
-                return self.executable.params[1:]
-            return self.executable.params
+        if self._executable.isinstance(er.Function):
+            if isinstance(self._executable, er.InstanceElement):
+                return self._executable.params[1:]
+            return self._executable.params
         else:
             try:
-                sub = self.executable.get_subscope_by_name('__init__')
+                sub = self._executable.get_subscope_by_name('__init__')
                 return sub.params[1:]  # ignore self
             except KeyError:
                 return []
@@ -292,7 +613,7 @@ class CallDef(object):
     def bracket_start(self):
         """ The indent of the bracket that is responsible for the last function
         call. """
-        c = self.call
+        c = self._call
         while c.next is not None:
             c = c.next
         return c.name.end_pos
@@ -300,12 +621,12 @@ class CallDef(object):
     @property
     def call_name(self):
         """ The name (e.g. 'isinstance') as a string. """
-        return str(self.executable.name)
+        return unicode(self._executable.name)
 
     @property
     def module(self):
-        return self.executable.get_parent_until()
+        return self._executable.get_parent_until()
 
     def __repr__(self):
-        return '<%s: %s index %s>' % (type(self).__name__, self.executable,
-                                    self.index)
+        return '<%s: %s index %s>' % (type(self).__name__, self._executable,
+                                      self.index)

jedi/builtin.py

@@ -1,5 +1,28 @@
+"""
+A big part of the Python standard libraries are unfortunately not only written
+in Python. The process works like this:
+
+- ``BuiltinModule`` imports the builtin module (e.g. ``sys``)
+- then ``BuiltinModule`` generates code with the docstrings of builtin
+  functions.
+- The :mod:`parsing` parser processes the generated code.
+
+This is possible, because many builtin functions supply docstrings, for example
+the method ``list.index`` has the following attribute ``__doc__``:
+
+    L.index(value, [start, [stop]]) -> integer -- return first index of value.
+        Raises ValueError if the value is not present.
+
+`PEP 257 <http://www.python.org/dev/peps/pep-0257/#one-line-docstrings>`_
+teaches how docstrings should look like for C functions.
+
+Additionally there's a ``Builtin``  instance in this module, to make it
+possible to access functions like ``list`` and ``int`` directly, the same way
+|jedi| access other functions.
+"""
+
 from __future__ import with_statement
-from _compatibility import exec_function, is_py3k
+from jedi._compatibility import exec_function, is_py3k
 
 import re
 import sys
@@ -9,71 +32,14 @@ if is_py3k:
 import types
 import inspect
 
-import debug
-import parsing
-import imports
+from jedi import common
+from jedi import debug
+from jedi import parsing
+from jedi import modules
 import evaluate
 
 
-def get_sys_path():
-    def check_virtual_env(sys_path):
-        """ Add virtualenv's site-packages to the `sys.path`."""
-        venv = os.getenv('VIRTUAL_ENV')
-        if not venv:
-            return
-        venv = os.path.abspath(venv)
-        p = os.path.join(
-            venv, 'lib', 'python%d.%d' % sys.version_info[:2], 'site-packages')
-        sys_path.insert(0, p)
-
-    p = sys.path[1:]
-    check_virtual_env(p)
-    return p
-
-
-class CachedModule(object):
-    """
-    The base type for all modules, which is not to be confused with
-    `parsing.Module`. Caching happens here.
-    """
-    cache = {}
-
-    def __init__(self, path=None, name=None):
-        self.path = path and os.path.abspath(path)
-        self.name = name
-        self._parser = None
-
-    @property
-    def parser(self):
-        """ get the parser lazy """
-        if not self._parser:
-            try:
-                timestamp, parser = self.cache[self.path or self.name]
-                if not self.path or os.path.getmtime(self.path) <= timestamp:
-                    self._parser = parser
-                else:
-                    # In case there is already a module cached and this module
-                    # has to be reparsed, we also need to invalidate the import
-                    # caches.
-                    imports.invalidate_star_import_cache(parser.module)
-                    raise KeyError()
-            except KeyError:
-                self._load_module()
-        return self._parser
-
-    def _get_source(self):
-        raise NotImplementedError()
-
-    def _load_module(self):
-        source = self._get_source()
-        self._parser = parsing.PyFuzzyParser(source, self.path or self.name)
-        p_time = None if not self.path else os.path.getmtime(self.path)
-
-        if self.path or self.name:
-            self.cache[self.path or self.name] = p_time, self._parser
-
-
-class Parser(CachedModule):
+class BuiltinModule(modules.CachedModule):
     """
     This module is a parser for all builtin modules, which are programmed in
     C/C++. It should also work on third party modules.
@@ -101,15 +67,13 @@ class Parser(CachedModule):
     if is_py3k:
         map_types['file object'] = 'import io; return io.TextIOWrapper()'
 
-    module_cache = {}
-
     def __init__(self, path=None, name=None, sys_path=None):
         if sys_path is None:
-            sys_path = get_sys_path()
+            sys_path = modules.get_sys_path()
         if not name:
             name = os.path.basename(path)
             name = name.rpartition('.')[0]  # cut file type (normally .so)
-        super(Parser, self).__init__(path=path, name=name)
+        super(BuiltinModule, self).__init__(path=path, name=name)
 
         self.sys_path = list(sys_path)
         self._module = None
@@ -197,10 +161,13 @@ class Parser(CachedModule):
 
         try:
             name = self.name
+            # sometimes there are stupid endings like `_sqlite3.cpython-32mu`
+            name = re.sub(r'\..*', '', name)
+
             if name == '__builtin__' and not is_py3k:
                 name = 'builtins'
             path = os.path.dirname(os.path.abspath(__file__))
-            with open(os.path.sep.join([path, 'mixin', name]) + '.py') as f:
+            with open(os.path.sep.join([path, 'mixin', name]) + '.pym') as f:
                 s = f.read()
         except IOError:
             return {}
@@ -214,15 +181,14 @@ class Parser(CachedModule):
 
 def _generate_code(scope, mixin_funcs={}, depth=0):
     """
-    Generate a string, which uses python syntax as an input to the
-    PyFuzzyParser.
+    Generate a string, which uses python syntax as an input to the Parser.
     """
     def get_doc(obj, indent=False):
         doc = inspect.getdoc(obj)
         if doc:
             doc = ('r"""\n%s\n"""\n' % doc)
             if indent:
-                doc = parsing.indent_block(doc)
+                doc = common.indent_block(doc)
             return doc
         return ''
 
@@ -267,9 +233,9 @@ def _generate_code(scope, mixin_funcs={}, depth=0):
                     if is_in_base_classes(scope, n, exe):
                         continue
                 if inspect.isbuiltin(exe) or inspect.ismethod(exe) \
-                            or inspect.ismethoddescriptor(exe):
+                        or inspect.ismethoddescriptor(exe):
                     funcs[n] = exe
-                elif inspect.isclass(exe):
+                elif inspect.isclass(exe) or inspect.ismodule(exe):
                     classes[n] = exe
                 elif inspect.ismemberdescriptor(exe):
                     members[n] = exe
@@ -288,14 +254,15 @@ def _generate_code(scope, mixin_funcs={}, depth=0):
     code += get_doc(scope)
 
     names = set(dir(scope)) - set(['__file__', '__name__', '__doc__',
-                                            '__path__', '__package__']) \
-                            | set(['mro'])
+                                   '__path__', '__package__']) \
+        | set(['mro'])
 
     classes, funcs, stmts, members = get_scope_objects(names)
 
     # classes
     for name, cl in classes.items():
-        bases = (c.__name__ for c in cl.__bases__)
+        bases = (c.__name__ for c in cl.__bases__) if inspect.isclass(cl) \
+            else []
         code += 'class %s(%s):\n' % (name, ','.join(bases))
         if depth == 0:
             try:
@@ -303,12 +270,12 @@ def _generate_code(scope, mixin_funcs={}, depth=0):
             except KeyError:
                 mixin = {}
             cl_code = _generate_code(cl, mixin, depth + 1)
-            code += parsing.indent_block(cl_code)
+            code += common.indent_block(cl_code)
         code += '\n'
 
     # functions
     for name, func in funcs.items():
-        params, ret = parse_function_doc(func)
+        params, ret = _parse_function_doc(func)
         if depth > 0:
             params = 'self, ' + params
         doc_str = get_doc(func, indent=True)
@@ -318,13 +285,23 @@ def _generate_code(scope, mixin_funcs={}, depth=0):
             # normal code generation
             code += 'def %s(%s):\n' % (name, params)
             code += doc_str
-            code += parsing.indent_block('%s\n\n' % ret)
+            code += common.indent_block('%s\n\n' % ret)
         else:
             # generation of code with mixins
             # the parser only supports basic functions with a newline after
             # the double dots
             # find doc_str place
-            pos = re.search(r'\):\s*\n', mixin).end()
+            try:
+                pos = re.search(r'\):\s*\n', mixin).end()
+            except TypeError:
+                # pypy uses a different reversed builtin
+                if name == 'reversed':
+                    mixin = 'def reversed(sequence):\n' \
+                            '    for i in self.__sequence: yield i'
+                    pos = 24
+                else:
+                    debug.warning('mixin trouble in pypy: %s', name)
+                    raise
             if pos is None:
                 raise Exception("Builtin function not parsed correctly")
             code += mixin[:pos] + doc_str + mixin[pos:]
@@ -336,7 +313,7 @@ def _generate_code(scope, mixin_funcs={}, depth=0):
             continue
         ret = 'pass'
         code += '@property\ndef %s(self):\n' % (name)
-        code += parsing.indent_block(get_doc(func) + '%s\n\n' % ret)
+        code += common.indent_block(get_doc(func) + '%s\n\n' % ret)
 
     # variables
     for name, value in stmts.items():
@@ -344,7 +321,7 @@ def _generate_code(scope, mixin_funcs={}, depth=0):
             file_type = io.TextIOWrapper
         else:
             file_type = types.FileType
-        if type(value) == file_type:
+        if isinstance(value, file_type):
             value = 'open()'
         elif name == 'None':
             value = ''
@@ -359,17 +336,10 @@ def _generate_code(scope, mixin_funcs={}, depth=0):
                 value = '%s.%s' % (mod, value)
         code += '%s = %s\n' % (name, value)
 
-    if depth == 0:
-        #with open('writeout.py', 'w') as f:
-        #    f.write(code)
-        #import sys
-        #sys.stdout.write(code)
-        #exit()
-        pass
     return code
 
 
-def parse_function_doc(func):
+def _parse_function_doc(func):
     """
     Takes a function and returns the params and return value as a tuple.
     This is nothing more than a docstring parser.
@@ -401,7 +371,7 @@ def parse_function_doc(func):
             return ','.join(args)
         while True:
             param_str, changes = re.subn(r' ?\[([^\[\]]+)\]',
-                                            change_options, param_str)
+                                         change_options, param_str)
             if changes == 0:
                 break
     except (ValueError, AttributeError):
@@ -423,7 +393,7 @@ def parse_function_doc(func):
         # New object -> object()
         ret_str = re.sub(r'[nN]ew (.*)', r'\1()', ret_str)
 
-        ret = Parser.map_types.get(ret_str, ret_str)
+        ret = BuiltinModule.map_types.get(ret_str, ret_str)
         if ret == ret_str and ret not in ['None', 'object', 'tuple', 'set']:
             debug.dbg('not working', ret_str)
         if ret != 'pass':
@@ -444,7 +414,7 @@ class Builtin(object):
     @property
     def builtin(self):
         if self._builtin is None:
-            self._builtin = Parser(name=self.name)
+            self._builtin = BuiltinModule(name=self.name)
         return self._builtin
 
     @property
@@ -460,7 +430,7 @@ class Builtin(object):
             class Container(object):
                 FunctionType = types.FunctionType
             source = _generate_code(Container, depth=0)
-            parser = parsing.PyFuzzyParser(source, None)
+            parser = parsing.Parser(source, None)
             module = parser.module
             module.parent = self.scope
             typ = evaluate.follow_path(iter(['FunctionType']), module, module)

jedi/cache.py

@@ -0,0 +1,326 @@
+"""
+This caching is very important for speed and memory optimizations. There's
+nothing really spectacular, just some decorators. The following cache types are
+available:
+
+- module caching (`load_module` and `save_module`), which uses pickle and is
+  really important to assure low load times of modules like ``numpy``.
+- the popular ``memoize_default`` works like a typical memoize and returns the
+  default otherwise.
+- ``CachedMetaClass`` uses ``memoize_default`` to do the same with classes.
+- ``time_cache`` can be used to cache something for just a limited time span,
+  which can be useful if there's user interaction and the user cannot react
+  faster than a certain time.
+
+This module is one of the reasons why |jedi| is not thread-safe. As you can see
+there are global variables, which are holding the cache information. Some of
+these variables are being cleaned after every API usage.
+"""
+from __future__ import with_statement
+
+import time
+import os
+import sys
+import json
+import hashlib
+try:
+    import cPickle as pickle
+except:
+    import pickle
+import shutil
+
+from jedi import settings
+from jedi import common
+from jedi import debug
+
+# memoize caches will be deleted after every action
+memoize_caches = []
+
+time_caches = []
+
+star_import_cache = {}
+
+# for fast_parser, should not be deleted
+parser_cache = {}
+
+
+class ParserCacheItem(object):
+    def __init__(self, parser, change_time=None):
+        self.parser = parser
+        if change_time is None:
+            change_time = time.time()
+        self.change_time = change_time
+
+
+def clear_caches(delete_all=False):
+    """ Jedi caches many things, that should be completed after each completion
+    finishes.
+
+    :param delete_all: Deletes also the cache that is normally not deleted,
+        like parser cache, which is important for faster parsing.
+    """
+    global memoize_caches, time_caches
+
+    # memorize_caches must never be deleted, because the dicts will get lost in
+    # the wrappers.
+    for m in memoize_caches:
+        m.clear()
+
+    if delete_all:
+        time_caches = []
+        star_import_cache.clear()
+        parser_cache.clear()
+    else:
+        # normally just kill the expired entries, not all
+        for tc in time_caches:
+            # check time_cache for expired entries
+            for key, (t, value) in list(tc.items()):
+                if t < time.time():
+                    # delete expired entries
+                    del tc[key]
+
+
+def memoize_default(default=None, cache=memoize_caches):
+    """ This is a typical memoization decorator, BUT there is one difference:
+    To prevent recursion it sets defaults.
+
+    Preventing recursion is in this case the much bigger use than speed. I
+    don't think, that there is a big speed difference, but there are many cases
+    where recursion could happen (think about a = b; b = a).
+    """
+    def func(function):
+        memo = {}
+        cache.append(memo)
+
+        def wrapper(*args, **kwargs):
+            key = (args, frozenset(kwargs.items()))
+            if key in memo:
+                return memo[key]
+            else:
+                memo[key] = default
+                rv = function(*args, **kwargs)
+                memo[key] = rv
+                return rv
+        return wrapper
+    return func
+
+
+class CachedMetaClass(type):
+    """ This is basically almost the same than the decorator above, it just
+    caches class initializations. I haven't found any other way, so I do it
+    with meta classes.
+    """
+    @memoize_default()
+    def __call__(self, *args, **kwargs):
+        return super(CachedMetaClass, self).__call__(*args, **kwargs)
+
+
+def time_cache(time_add_setting):
+    """ This decorator works as follows: Call it with a setting and after that
+    use the function with a callable that returns the key.
+    But: This function is only called if the key is not available. After a
+    certain amount of time (`time_add_setting`) the cache is invalid.
+    """
+    def _temp(key_func):
+        dct = {}
+        time_caches.append(dct)
+
+        def wrapper(optional_callable, *args, **kwargs):
+            key = key_func(*args, **kwargs)
+            value = None
+            if key in dct:
+                expiry, value = dct[key]
+                if expiry > time.time():
+                    return value
+            value = optional_callable()
+            time_add = getattr(settings, time_add_setting)
+            if key is not None:
+                dct[key] = time.time() + time_add, value
+            return value
+        return wrapper
+    return _temp
+
+
+@time_cache("function_definition_validity")
+def cache_function_definition(stmt):
+    module_path = stmt.get_parent_until().path
+    return None if module_path is None else (module_path, stmt.start_pos)
+
+
+def cache_star_import(func):
+    def wrapper(scope, *args, **kwargs):
+        with common.ignored(KeyError):
+            mods = star_import_cache[scope]
+            if mods[0] + settings.star_import_cache_validity > time.time():
+                return mods[1]
+        # cache is too old and therefore invalid or not available
+        invalidate_star_import_cache(scope)
+        mods = func(scope, *args, **kwargs)
+        star_import_cache[scope] = time.time(), mods
+
+        return mods
+    return wrapper
+
+
+def invalidate_star_import_cache(module, only_main=False):
+    """ Important if some new modules are being reparsed """
+    with common.ignored(KeyError):
+        t, mods = star_import_cache[module]
+
+        del star_import_cache[module]
+
+        for m in mods:
+            invalidate_star_import_cache(m, only_main=True)
+
+    if not only_main:
+        # We need a list here because otherwise the list is being changed
+        # during the iteration in py3k: iteritems -> items.
+        for key, (t, mods) in list(star_import_cache.items()):
+            if module in mods:
+                invalidate_star_import_cache(key)
+
+
+def load_module(path, name):
+    """
+    Returns the module or None, if it fails.
+    """
+    if path is None and name is None:
+        return None
+
+    tim = os.path.getmtime(path) if path else None
+    n = name if path is None else path
+    try:
+        parser_cache_item = parser_cache[n]
+        if not path or tim <= parser_cache_item.change_time:
+            return parser_cache_item.parser
+        else:
+            # In case there is already a module cached and this module
+            # has to be reparsed, we also need to invalidate the import
+            # caches.
+            invalidate_star_import_cache(parser_cache_item.parser.module)
+    except KeyError:
+        if settings.use_filesystem_cache:
+            return ModulePickling.load_module(n, tim)
+
+
+def save_module(path, name, parser, pickling=True):
+    try:
+        p_time = None if not path else os.path.getmtime(path)
+    except OSError:
+        p_time = None
+        pickling = False
+
+    n = name if path is None else path
+    item = ParserCacheItem(parser, p_time)
+    parser_cache[n] = item
+    if settings.use_filesystem_cache and pickling:
+        ModulePickling.save_module(n, item)
+
+
+class _ModulePickling(object):
+
+    version = 3
+    """
+    Version number (integer) for file system cache.
+
+    Increment this number when there are any incompatible changes in
+    parser representation classes.  For example, the following changes
+    are regarded as incompatible.
+
+    - Class name is changed.
+    - Class is moved to another module.
+    - Defined slot of the class is changed.
+    """
+
+    def __init__(self):
+        self.__index = None
+        self.py_tag = 'cpython-%s%s' % sys.version_info[:2]
+        """
+        Short name for distinguish Python implementations and versions.
+
+        It's like `sys.implementation.cache_tag` but for Python < 3.3
+        we generate something similar.  See:
+        http://docs.python.org/3/library/sys.html#sys.implementation
+
+        .. todo:: Detect interpreter (e.g., PyPy).
+        """
+
+    def load_module(self, path, original_changed_time):
+        try:
+            pickle_changed_time = self._index[path]
+        except KeyError:
+            return None
+        if original_changed_time is not None \
+                and pickle_changed_time < original_changed_time:
+            # the pickle file is outdated
+            return None
+
+        with open(self._get_hashed_path(path), 'rb') as f:
+            parser_cache_item = pickle.load(f)
+
+        debug.dbg('pickle loaded', path)
+        parser_cache[path] = parser_cache_item
+        return parser_cache_item.parser
+
+    def save_module(self, path, parser_cache_item):
+        self.__index = None
+        try:
+            files = self._index
+        except KeyError:
+            files = {}
+            self._index = files
+
+        with open(self._get_hashed_path(path), 'wb') as f:
+            pickle.dump(parser_cache_item, f, pickle.HIGHEST_PROTOCOL)
+            files[path] = parser_cache_item.change_time
+
+        self._flush_index()
+
+    @property
+    def _index(self):
+        if self.__index is None:
+            try:
+                with open(self._get_path('index.json')) as f:
+                    data = json.load(f)
+            except (IOError, ValueError):
+                self.__index = {}
+            else:
+                # 0 means version is not defined (= always delete cache):
+                if data.get('version', 0) != self.version:
+                    self.delete_cache()
+                    self.__index = {}
+                else:
+                    self.__index = data['index']
+        return self.__index
+
+    def _remove_old_modules(self):
+        # TODO use
+        change = False
+        if change:
+            self._flush_index(self)
+            self._index  # reload index
+
+    def _flush_index(self):
+        data = {'version': self.version, 'index': self._index}
+        with open(self._get_path('index.json'), 'w') as f:
+            json.dump(data, f)
+        self.__index = None
+
+    def delete_cache(self):
+        shutil.rmtree(self._cache_directory())
+
+    def _get_hashed_path(self, path):
+        return self._get_path('%s.pkl' % hashlib.md5(path.encode("utf-8")).hexdigest())
+
+    def _get_path(self, file):
+        dir = self._cache_directory()
+        if not os.path.exists(dir):
+            os.makedirs(dir)
+        return os.path.join(dir, file)
+
+    def _cache_directory(self):
+        return os.path.join(settings.cache_directory, self.py_tag)
+
+
+# is a singleton
+ModulePickling = _ModulePickling()

jedi/common.py

@@ -0,0 +1,211 @@
+""" A universal module with functions / classes without dependencies. """
+import sys
+import contextlib
+import functools
+import tokenizer as tokenize
+
+from jedi._compatibility import next, reraise
+from jedi import settings
+
+FLOWS = ['if', 'else', 'elif', 'while', 'with', 'try', 'except', 'finally']
+
+
+class MultiLevelStopIteration(Exception):
+    """
+    StopIteration's get catched pretty easy by for loops, let errors propagate.
+    """
+    pass
+
+
+class UncaughtAttributeError(Exception):
+    """
+    Important, because `__getattr__` and `hasattr` catch AttributeErrors
+    implicitly. This is really evil (mainly because of `__getattr__`).
+    `hasattr` in Python 2 is even more evil, because it catches ALL exceptions.
+    Therefore this class originally had to be derived from `BaseException`
+    instead of `Exception`.  But because I removed relevant `hasattr` from
+    the code base, we can now switch back to `Exception`.
+
+    :param base: return values of sys.exc_info().
+    """
+
+
+def rethrow_uncaught(func):
+    """
+    Re-throw uncaught `AttributeError`.
+
+    Usage:  Put ``@rethrow_uncaught`` in front of the function
+    which does **not** suppose to raise `AttributeError`.
+
+    AttributeError is easily get caught by `hasattr` and another
+    ``except AttributeError`` clause.  This becomes problem when you use
+    a lot of "dynamic" attributes (e.g., using ``@property``) because you
+    can't distinguish if the property does not exist for real or some code
+    inside of the "dynamic" attribute through that error.  In a well
+    written code, such error should not exist but getting there is very
+    difficult.  This decorator is to help us getting there by changing
+    `AttributeError` to `UncaughtAttributeError` to avoid unexpected catch.
+    This helps us noticing bugs earlier and facilitates debugging.
+
+    .. note:: Treating StopIteration here is easy.
+              Add that feature when needed.
+    """
+    @functools.wraps(func)
+    def wrapper(*args, **kwds):
+        try:
+            return func(*args, **kwds)
+        except AttributeError:
+            exc_info = sys.exc_info()
+            reraise(UncaughtAttributeError(exc_info[1]), exc_info[2])
+    return wrapper
+
+
+class PushBackIterator(object):
+    def __init__(self, iterator):
+        self.pushes = []
+        self.iterator = iterator
+        self.current = None
+
+    def push_back(self, value):
+        self.pushes.append(value)
+
+    def __iter__(self):
+        return self
+
+    def next(self):
+        """ Python 2 Compatibility """
+        return self.__next__()
+
+    def __next__(self):
+        if self.pushes:
+            self.current = self.pushes.pop()
+        else:
+            self.current = next(self.iterator)
+        return self.current
+
+
+class NoErrorTokenizer(object):
+    def __init__(self, readline, offset=(0, 0), is_fast_parser=False):
+        self.readline = readline
+        self.gen = tokenize.generate_tokens(readline)
+        self.offset = offset
+        self.closed = False
+        self.is_first = True
+        self.push_backs = []
+
+        # fast parser options
+        self.is_fast_parser = is_fast_parser
+        self.current = self.previous = [None, None, (0, 0), (0, 0), '']
+        self.in_flow = False
+        self.new_indent = False
+        self.parser_indent = self.old_parser_indent = 0
+        self.is_decorator = False
+        self.first_stmt = True
+
+    def push_last_back(self):
+        self.push_backs.append(self.current)
+
+    def next(self):
+        """ Python 2 Compatibility """
+        return self.__next__()
+
+    def __next__(self):
+        if self.closed:
+            raise MultiLevelStopIteration()
+        if self.push_backs:
+            return self.push_backs.pop(0)
+
+        self.last_previous = self.previous
+        self.previous = self.current
+        self.current = next(self.gen)
+        c = list(self.current)
+
+        if c[0] == tokenize.ENDMARKER:
+            self.current = self.previous
+            self.previous = self.last_previous
+            raise MultiLevelStopIteration()
+
+        # this is exactly the same check as in fast_parser, but this time with
+        # tokenize and therefore precise.
+        breaks = ['def', 'class', '@']
+
+        if self.is_first:
+            c[2] = self.offset[0] + c[2][0], self.offset[1] + c[2][1]
+            c[3] = self.offset[0] + c[3][0], self.offset[1] + c[3][1]
+            self.is_first = False
+        else:
+            c[2] = self.offset[0] + c[2][0], c[2][1]
+            c[3] = self.offset[0] + c[3][0], c[3][1]
+        self.current = c
+
+        def close():
+            if not self.first_stmt:
+                self.closed = True
+                raise MultiLevelStopIteration()
+        # ignore indents/comments
+        if self.is_fast_parser \
+                and self.previous[0] in (tokenize.INDENT, tokenize.NL, None,
+                                         tokenize.NEWLINE, tokenize.DEDENT) \
+                and c[0] not in (tokenize.COMMENT, tokenize.INDENT,
+                                 tokenize.NL, tokenize.NEWLINE, tokenize.DEDENT):
+            # print c, tokenize.tok_name[c[0]]
+
+            tok = c[1]
+            indent = c[2][1]
+            if indent < self.parser_indent:  # -> dedent
+                self.parser_indent = indent
+                self.new_indent = False
+                if not self.in_flow or indent < self.old_parser_indent:
+                    close()
+                self.in_flow = False
+            elif self.new_indent:
+                self.parser_indent = indent
+                self.new_indent = False
+
+            if not self.in_flow:
+                if tok in FLOWS or tok in breaks:
+                    self.in_flow = tok in FLOWS
+                    if not self.is_decorator and not self.in_flow:
+                        close()
+                    self.is_decorator = '@' == tok
+                    if not self.is_decorator:
+                        self.old_parser_indent = self.parser_indent
+                        self.parser_indent += 1  # new scope: must be higher
+                        self.new_indent = True
+
+            if tok != '@':
+                if self.first_stmt and not self.new_indent:
+                    self.parser_indent = indent
+                self.first_stmt = False
+        return c
+
+
+@contextlib.contextmanager
+def scale_speed_settings(factor):
+    a = settings.max_executions
+    b = settings.max_until_execution_unique
+    settings.max_executions *= factor
+    settings.max_until_execution_unique *= factor
+    yield
+    settings.max_executions = a
+    settings.max_until_execution_unique = b
+
+
+def indent_block(text, indention='    '):
+    """ This function indents a text block with a default of four spaces """
+    temp = ''
+    while text and text[-1] == '\n':
+        temp += text[-1]
+        text = text[:-1]
+    lines = text.split('\n')
+    return '\n'.join(map(lambda s: indention + s, lines)) + temp
+
+
+@contextlib.contextmanager
+def ignored(*exceptions):
+    """Context manager that ignores all of the specified exceptions. This will
+    be in the standard library starting with Python 3.4."""
+    try:
+        yield
+    except exceptions:
+        pass

jedi/debug.py

@@ -1,3 +1,4 @@
+from _compatibility import u, encoding, is_py3k
 import inspect
 import time
 
@@ -36,12 +37,12 @@ def dbg(*args):
         frm = inspect.stack()[1]
         mod = inspect.getmodule(frm[0])
         if not (mod.__name__ in ignored_modules):
-            debug_function(NOTICE, 'dbg: ' + ', '.join(str(a) for a in args))
+            debug_function(NOTICE, 'dbg: ' + ', '.join(u(a) for a in args))
 
 
 def warning(*args):
     if debug_function and enable_warning:
-        debug_function(WARNING, 'warning: ' + ', '.join(str(a) for a in args))
+        debug_function(WARNING, 'warning: ' + ', '.join(u(a) for a in args))
 
 
 def speed(name):
@@ -58,7 +59,9 @@ def print_to_stdout(level, str_out):
         col = Fore.RED
     else:
         col = Fore.YELLOW
+    if not is_py3k:
+        str_out = str_out.encode(encoding, 'replace')
     print(col + str_out + Fore.RESET)
 
 
-#debug_function = print_to_stdout
+# debug_function = print_to_stdout

jedi/docstrings.py

@@ -1,34 +1,131 @@
-""" Processing of docstrings, which means parsing for types. """
+"""
+Docstrings are another source of information for functions and classes.
+:mod:`dynamic` tries to find all executions of functions, while the docstring
+parsing is much easier. There are two different types of docstrings that |jedi|
+understands:
+
+- `Sphinx <http://sphinx-doc.org/markup/desc.html#info-field-lists>`_
+- `Epydoc <http://epydoc.sourceforge.net/manual-fields.html>`_
+
+For example, the sphinx annotation ``:type foo: str`` clearly states that the
+type of ``foo`` is ``str``.
+
+As an addition to parameter searching, this module also provides return
+annotations.
+"""
 
 import re
 
+from jedi import cache
+from jedi import parsing
 import evaluate
+import evaluate_representation as er
+
+DOCSTRING_PARAM_PATTERNS = [
+    r'\s*:type\s+%s:\s*([^\n]+)',  # Sphinx
+    r'\s*@type\s+%s:\s*([^\n]+)',  # Epydoc
+]
+
+DOCSTRING_RETURN_PATTERNS = [
+    re.compile(r'\s*:rtype:\s*([^\n]+)', re.M),  # Sphinx
+    re.compile(r'\s*@rtype:\s*([^\n]+)', re.M),  # Epydoc
+]
+
+REST_ROLE_PATTERN = re.compile(r':[^`]+:`([^`]+)`')
 
 
-#@evaluate.memoize_default()  # TODO add
+@cache.memoize_default()
 def follow_param(param):
     func = param.parent_function
-    #print func, param, param.parent_function
-    param_str = search_param_in_docstr(func.docstr, str(param.get_name()))
+    # print func, param, param.parent_function
+    param_str = _search_param_in_docstr(func.docstr, str(param.get_name()))
+    user_position = (1, 0)
 
     if param_str is not None:
-        scope = func.get_parent_until()
-        return evaluate.get_scopes_for_name(scope, param_str,
-                                            search_global=True)
+
+        # Try to import module part in dotted name.
+        # (e.g., 'threading' in 'threading.Thread').
+        if '.' in param_str:
+            param_str = 'import %s\n%s' % (
+                param_str.rsplit('.', 1)[0],
+                param_str)
+            user_position = (2, 0)
+
+        p = parsing.Parser(param_str, None, user_position,
+                           no_docstr=True)
+        if p.user_stmt is None:
+            return []
+        return evaluate.follow_statement(p.user_stmt)
     return []
 
 
-def search_param_in_docstr(docstr, param_str):
-    lines = docstr.split('\n')
+def _search_param_in_docstr(docstr, param_str):
+    """
+    Search `docstr` for a type of `param_str`.
 
-    # look at #40 to see definitions of those params 
-    sphinx_comp = ':type %s:' % param_str
-    googley_comp = re.compile('\s*%s\s+\(([^()]+)\)' % re.escape(param_str))
-    for l in lines:
-        if l.startswith(sphinx_comp):
-            return l.replace(sphinx_comp, '', 1).strip()
+    >>> _search_param_in_docstr(':type param: int', 'param')
+    'int'
+    >>> _search_param_in_docstr('@type param: int', 'param')
+    'int'
+    >>> _search_param_in_docstr(
+    ...   ':type param: :class:`threading.Thread`', 'param')
+    'threading.Thread'
+    >>> _search_param_in_docstr('no document', 'param') is None
+    True
+
+    """
+    # look at #40 to see definitions of those params
+    patterns = [re.compile(p % re.escape(param_str))
+                for p in DOCSTRING_PARAM_PATTERNS]
+    for pattern in patterns:
+        match = pattern.search(docstr)
+        if match:
+            return _strip_rest_role(match.group(1))
 
-        r = re.match(googley_comp, l)
-        if r is not None:
-            return r.group(1)
     return None
+
+
+def _strip_rest_role(type_str):
+    """
+    Strip off the part looks like a ReST role in `type_str`.
+
+    >>> _strip_rest_role(':class:`ClassName`')  # strip off :class:
+    'ClassName'
+    >>> _strip_rest_role(':py:obj:`module.Object`')  # works with domain
+    'module.Object'
+    >>> _strip_rest_role('ClassName')  # do nothing when not ReST role
+    'ClassName'
+
+    See also:
+    http://sphinx-doc.org/domains.html#cross-referencing-python-objects
+
+    """
+    match = REST_ROLE_PATTERN.match(type_str)
+    if match:
+        return match.group(1)
+    else:
+        return type_str
+
+
+def find_return_types(func):
+    def search_return_in_docstr(code):
+        for p in DOCSTRING_RETURN_PATTERNS:
+            match = p.search(code)
+            if match:
+                return match.group(1)
+
+    if isinstance(func, er.InstanceElement):
+        func = func.var
+
+    if isinstance(func, er.Function):
+        func = func.base_func
+
+    type_str = search_return_in_docstr(func.docstr)
+    if not type_str:
+        return []
+
+    p = parsing.Parser(type_str, None, (1, 0), no_docstr=True)
+    if p.user_stmt is None:
+        return []
+    p.user_stmt.parent = func
+    return list(evaluate.follow_statement(p.user_stmt))

jedi/dynamic.py

@@ -1,23 +1,71 @@
 """
-For dynamic completion.
+To understand Python on a deeper level, |jedi| needs to understand some of the
+dynamic features of Python, however this probably the most complicated part:
 
-Sorry to everyone who is reading this code. Especially the array parts are
-really cryptic and not understandable. It's just a hack, that turned out to be
-working quite good.
+- Array modifications (e.g. ``list.append``)
+- Parameter completion in functions
+- Flow checks (e.g. ``if isinstance(a, str)`` -> a is a str)
+
+Array modifications
+*******************
+
+If the content of an array (``set``/``list``) is wanted somewhere, the current
+module will be checked for appearances of ``arr.append``, ``arr.insert``, etc.
+If the ``arr`` name points to an actual array, the content will be added
+
+This can be really cpu intensive, as you can imagine. Because |jedi| has to
+follow **every** ``append``. However this works pretty good, because in *slow*
+cases, the recursion detector and other settings will stop this process.
+
+It is important to note that:
+
+1. Array modfications work only in the current module
+2. Only Array additions are being checked, ``list.pop``, etc. is being ignored.
+
+Parameter completion
+********************
+
+One of the really important features of |jedi| is to have an option to
+understand code like this::
+
+    def foo(bar):
+        bar. # completion here
+    foo(1)
+
+There's no doubt wheter bar is an ``int`` or not, but if there's also a call
+like ``foo('str')``, what would happen? Well, we'll just show both. Because
+that's what a human would expect.
+
+It works as follows:
+
+- A param is being encountered
+- search for function calls named ``foo``
+- execute these calls and check the injected params. This work with a
+  ``ParamListener``.
+
+Flow checks
+***********
+
+Flow checks are not really mature. There's only a check for ``isinstance``.  It
+would check whether a flow has the form of ``if isinstance(a, type_or_tuple)``.
+Unfortunately every other thing is being ignored (e.g. a == '' would be easy to
+check for -> a is a string). There's big potential in these checks.
 """
 from __future__ import with_statement
 
 import os
 
-import parsing
-import modules
-import evaluate
-import helpers
-import settings
-import debug
-import builtin
-import imports
+from jedi import cache
+from jedi import parsing_representation as pr
+from jedi import modules
+from jedi import settings
+from jedi import common
+from jedi import debug
+from jedi import fast_parser
 import api_classes
+import evaluate
+import imports
+import evaluate_representation as er
 
 # This is something like the sys.path, but only for searching params. It means
 # that this is the order in which Jedi searches params.
@@ -31,7 +79,7 @@ def get_directory_modules_for_name(mods, name):
     """
     def check_python_file(path):
         try:
-            return builtin.CachedModule.cache[path][1].module
+            return cache.parser_cache[path].parser.module
         except KeyError:
             try:
                 return check_fs(path)
@@ -40,12 +88,12 @@ def get_directory_modules_for_name(mods, name):
 
     def check_fs(path):
         with open(path) as f:
-            source = f.read()
+            source = modules.source_to_unicode(f.read())
             if name in source:
                 return modules.Module(path, source).parser.module
 
     # skip non python modules
-    mods = set(m for m in mods if m.path.endswith('.py'))
+    mods = set(m for m in mods if m.path is None or m.path.endswith('.py'))
     mod_paths = set()
     for m in mods:
         mod_paths.add(m.path)
@@ -54,13 +102,15 @@ def get_directory_modules_for_name(mods, name):
     if settings.dynamic_params_for_other_modules:
         paths = set(settings.additional_dynamic_modules)
         for p in mod_paths:
-            d = os.path.dirname(p)
-            for entry in os.listdir(d):
-                if entry not in mod_paths:
-                    if entry.endswith('.py'):
-                        paths.add(d + os.path.sep + entry)
+            if p is not None:
+                d = os.path.dirname(p)
+                for entry in os.listdir(d):
+                    if entry not in mod_paths:
+                        if entry.endswith('.py'):
+                            paths.add(d + os.path.sep + entry)
 
-        for p in paths:
+        for p in sorted(paths):
+            # make testing easier, sort it - same results on every interpreter
             c = check_python_file(p)
             if c is not None and c not in mods:
                 yield c
@@ -93,7 +143,7 @@ class ParamListener(object):
         self.param_possibilities.append(params)
 
 
-@evaluate.memoize_default([])
+@cache.memoize_default([])
 def search_params(param):
     """
     This is a dynamic search for params. If you try to complete a type:
@@ -122,14 +172,46 @@ def search_params(param):
                 return []
 
             for stmt in possible_stmts:
-                if not isinstance(stmt, parsing.Import):
-                    calls = _scan_array(stmt.get_assignment_calls(), func_name)
-                    for c in calls:
-                        # no execution means that params cannot be set
-                        call_path = c.generate_call_path()
-                        pos = c.start_pos
-                        scope = stmt.parent
-                        evaluate.follow_call_path(call_path, scope, pos)
+                if isinstance(stmt, pr.Import):
+                    continue
+                calls = _scan_statement(stmt, func_name)
+                for c in calls:
+                    # no execution means that params cannot be set
+                    call_path = list(c.generate_call_path())
+                    pos = c.start_pos
+                    scope = stmt.parent
+
+                    # this whole stuff is just to not execute certain parts
+                    # (speed improvement), basically we could just call
+                    # ``follow_call_path`` on the call_path and it would
+                    # also work.
+                    def listRightIndex(lst, value):
+                        return len(lst) - lst[-1::-1].index(value) -1
+
+                    # Need to take right index, because there could be a
+                    # func usage before.
+                    i = listRightIndex(call_path, func_name)
+                    first, last = call_path[:i], call_path[i+1:]
+                    if not last and not call_path.index(func_name) != i:
+                        continue
+                    scopes = [scope]
+                    if first:
+                        scopes = evaluate.follow_call_path(iter(first), scope, pos)
+                        pos = None
+                    for scope in scopes:
+                        s = evaluate.find_name(scope, func_name, position=pos,
+                                               search_global=not first,
+                                               resolve_decorator=False)
+
+                        c = [getattr(escope, 'base_func', None) or escope.base
+                            for escope in s
+                            if escope.isinstance(er.Function, er.Class)
+                        ]
+                        if compare in c:
+                            # only if we have the correct function we execute
+                            # it, otherwise just ignore it.
+                            evaluate.follow_paths(iter(last), s, scope)
+
             return listener.param_possibilities
 
         result = []
@@ -139,19 +221,22 @@ def search_params(param):
                     result += evaluate.follow_statement(p.parent)
         return result
 
-    func = param.get_parent_until(parsing.Function)
+    func = param.get_parent_until(pr.Function)
     current_module = param.get_parent_until()
     func_name = str(func.name)
-    if func_name == '__init__' and isinstance(func.parent, parsing.Class):
+    compare = func
+    if func_name == '__init__' and isinstance(func.parent, pr.Class):
         func_name = str(func.parent.name)
+        compare = func.parent
 
     # get the param name
     if param.assignment_details:
-        arr = param.assignment_details[0][1]
+        # first assignment details, others would be a syntax error
+        commands, op = param.assignment_details[0]
     else:
-        arr = param.get_assignment_calls()
-    offset = 1 if arr[0][0] in ['*', '**'] else 0
-    param_name = str(arr[0][offset].name)
+        commands = param.get_commands()
+    offset = 1 if commands[0] in ['*', '**'] else 0
+    param_name = str(commands[offset].name)
 
     # add the listener
     listener = ParamListener()
@@ -172,61 +257,56 @@ def search_params(param):
 
 def check_array_additions(array):
     """ Just a mapper function for the internal _check_array_additions """
-    if array._array.type not in ['list', 'set']:
+    if not pr.Array.is_type(array._array, pr.Array.LIST, pr.Array.SET):
         # TODO also check for dict updates
         return []
 
     is_list = array._array.type == 'list'
-    current_module = array._array.parent_stmt.get_parent_until()
+    current_module = array._array.get_parent_until()
     res = _check_array_additions(array, current_module, is_list)
     return res
 
 
-def _scan_array(arr, search_name):
+def _scan_statement(stmt, search_name, assignment_details=False):
     """ Returns the function Call that match search_name in an Array. """
-    result = []
-    for sub in arr:
-        for s in sub:
-            if isinstance(s, parsing.Array):
-                result += _scan_array(s, search_name)
-            elif isinstance(s, parsing.Call):
-                s_new = s
-                while s_new is not None:
-                    n = s_new.name
-                    if isinstance(n, parsing.Name) and search_name in n.names:
-                        result.append(s)
+    def scan_array(arr, search_name):
+        result = []
+        if arr.type == pr.Array.DICT:
+            for key_stmt, value_stmt in arr.items():
+                result += _scan_statement(key_stmt, search_name)
+                result += _scan_statement(value_stmt, search_name)
+        else:
+            for stmt in arr:
+                result += _scan_statement(stmt, search_name)
+        return result
+
+    check = list(stmt.get_commands())
+    if assignment_details:
+        for commands, op in stmt.assignment_details:
+            check += commands
+
+    result = []
+    for c in check:
+        if isinstance(c, pr.Array):
+            result += scan_array(c, search_name)
+        elif isinstance(c, pr.Call):
+            s_new = c
+            while s_new is not None:
+                n = s_new.name
+                if isinstance(n, pr.Name) and search_name in n.names:
+                    result.append(c)
+
+                if s_new.execution is not None:
+                    result += scan_array(s_new.execution, search_name)
+                s_new = s_new.next
 
-                    if s_new.execution is not None:
-                        result += _scan_array(s_new.execution, search_name)
-                    s_new = s_new.next
     return result
 
 
-counter = 0
-def dec(func):
-    """ TODO delete this """
-    def wrapper(*args, **kwargs):
-        global counter
-        element = args[0]
-        if isinstance(element, evaluate.Array):
-            stmt = element._array.parent_stmt
-        else:
-            # must be instance
-            stmt = element.var_args.parent_stmt
-        print('  ' * counter + 'recursion,', stmt)
-        counter += 1
-        res = func(*args, **kwargs)
-        counter -= 1
-        #print '  '*counter + 'end,'
-        return res
-    return wrapper
-
-
-#@dec
-@evaluate.memoize_default([])
+@cache.memoize_default([])
 def _check_array_additions(compare_array, module, is_list):
     """
-    Checks if a `parsing.Array` has "add" statements:
+    Checks if a `pr.Array` has "add" statements:
     >>> a = [""]
     >>> a.append(1)
     """
@@ -249,7 +329,7 @@ def _check_array_additions(compare_array, module, is_list):
             backtrack_path = iter(call_path[:separate_index])
 
             position = c.start_pos
-            scope = c.parent_stmt.parent
+            scope = c.get_parent_until(pr.IsScope)
 
             found = evaluate.follow_call_path(backtrack_path, scope, position)
             if not compare_array in found:
@@ -259,36 +339,39 @@ def _check_array_additions(compare_array, module, is_list):
             if not params.values:
                 continue  # no params: just ignore it
             if add_name in ['append', 'add']:
-                result += evaluate.follow_call_list(params)
+                for param in params:
+                    result += evaluate.follow_statement(param)
             elif add_name in ['insert']:
                 try:
                     second_param = params[1]
                 except IndexError:
                     continue
                 else:
-                    result += evaluate.follow_call_list([second_param])
+                    result += evaluate.follow_statement(second_param)
             elif add_name in ['extend', 'update']:
-                iterators = evaluate.follow_call_list(params)
+                for param in params:
+                    iterators = evaluate.follow_statement(param)
                 result += evaluate.get_iterator_types(iterators)
         return result
 
     def get_execution_parent(element, *stop_classes):
         """ Used to get an Instance/Execution parent """
-        if isinstance(element, evaluate.Array):
-            stmt = element._array.parent_stmt
+        if isinstance(element, er.Array):
+            stmt = element._array.parent
         else:
-            # must be instance
-            stmt = element.var_args.parent_stmt
-        if isinstance(stmt, evaluate.InstanceElement):
-            stop_classes = list(stop_classes) + [evaluate.Function]
+            # is an Instance with an ArrayInstance inside
+            stmt = element.var_args[0].var_args.parent
+        if isinstance(stmt, er.InstanceElement):
+            stop_classes = list(stop_classes) + [er.Function]
         return stmt.get_parent_until(stop_classes)
 
     temp_param_add = settings.dynamic_params_for_other_modules
     settings.dynamic_params_for_other_modules = False
 
     search_names = ['append', 'extend', 'insert'] if is_list else \
-                                                            ['add', 'update']
-    comp_arr_parent = get_execution_parent(compare_array, evaluate.Execution)
+        ['add', 'update']
+    comp_arr_parent = get_execution_parent(compare_array, er.Execution)
+
     possible_stmts = []
     res = []
     for n in search_names:
@@ -301,20 +384,20 @@ def _check_array_additions(compare_array, module, is_list):
             # can search for the same statement, that is in the module
             # dict. Executions are somewhat special in jedi, since they
             # literally copy the contents of a function.
-            if isinstance(comp_arr_parent, evaluate.Execution):
+            if isinstance(comp_arr_parent, er.Execution):
                 stmt = comp_arr_parent. \
-                                get_statement_for_position(stmt.start_pos)
+                    get_statement_for_position(stmt.start_pos)
                 if stmt is None:
                     continue
             # InstanceElements are special, because they don't get copied,
             # but have this wrapper around them.
-            if isinstance(comp_arr_parent, evaluate.InstanceElement):
-                stmt = evaluate.InstanceElement(comp_arr_parent.instance, stmt)
+            if isinstance(comp_arr_parent, er.InstanceElement):
+                stmt = er.InstanceElement(comp_arr_parent.instance, stmt)
 
             if evaluate.follow_statement.push_stmt(stmt):
                 # check recursion
                 continue
-            res += check_calls(_scan_array(stmt.get_assignment_calls(), n), n)
+            res += check_calls(_scan_statement(stmt, n), n)
             evaluate.follow_statement.pop_stmt()
     # reset settings
     settings.dynamic_params_for_other_modules = temp_param_add
@@ -322,14 +405,14 @@ def _check_array_additions(compare_array, module, is_list):
 
 
 def check_array_instances(instance):
-    """ Used for set() and list() instances. """
+    """Used for set() and list() instances."""
     if not settings.dynamic_arrays_instances:
         return instance.var_args
     ai = ArrayInstance(instance)
-    return helpers.generate_param_array([ai], instance.var_args.parent_stmt)
+    return [ai]
 
 
-class ArrayInstance(parsing.Base):
+class ArrayInstance(pr.Base):
     """
     Used for the usage of set() and list().
     This is definitely a hack, but a good one :-)
@@ -345,66 +428,68 @@ class ArrayInstance(parsing.Base):
         lists/sets are too complicated too handle that.
         """
         items = []
-        for array in evaluate.follow_call_list(self.var_args):
-            if isinstance(array, evaluate.Instance) and len(array.var_args):
-                temp = array.var_args[0][0]
-                if isinstance(temp, ArrayInstance):
-                    # prevent recursions
-                    # TODO compare Modules
-                    if self.var_args.start_pos != temp.var_args.start_pos:
-                        items += temp.iter_content()
-                    else:
-                        debug.warning('ArrayInstance recursion', self.var_args)
-                    continue
-            items += evaluate.get_iterator_types([array])
+        for stmt in self.var_args:
+            for typ in evaluate.follow_statement(stmt):
+                if isinstance(typ, er.Instance) and len(typ.var_args):
+                    array = typ.var_args[0]
+                    if isinstance(array, ArrayInstance):
+                        # prevent recursions
+                        # TODO compare Modules
+                        if self.var_args.start_pos != array.var_args.start_pos:
+                            items += array.iter_content()
+                        else:
+                            debug.warning(
+                                'ArrayInstance recursion',
+                                self.var_args)
+                        continue
+                items += evaluate.get_iterator_types([typ])
 
-        if self.var_args.parent_stmt is None:
+        # TODO check if exclusion of tuple is a problem here.
+        if isinstance(self.var_args, tuple) or self.var_args.parent is None:
             return []  # generated var_args should not be checked for arrays
 
-        module = self.var_args.parent_stmt.get_parent_until()
+        module = self.var_args.get_parent_until()
         is_list = str(self.instance.name) == 'list'
         items += _check_array_additions(self.instance, module, is_list)
         return items
 
 
-def related_names(definitions, search_name, mods):
+def usages(definitions, search_name, mods):
+    def compare_array(definitions):
+        """ `definitions` are being compared by module/start_pos, because
+        sometimes the id's of the objects change (e.g. executions).
+        """
+        result = []
+        for d in definitions:
+            module = d.get_parent_until()
+            result.append((module, d.start_pos))
+        return result
+
     def check_call(call):
         result = []
         follow = []  # There might be multiple search_name's in one call_path
         call_path = list(call.generate_call_path())
         for i, name in enumerate(call_path):
-            # name is `parsing.NamePart`.
+            # name is `pr.NamePart`.
             if name == search_name:
                 follow.append(call_path[:i + 1])
 
         for f in follow:
-            follow_res, search = evaluate.goto(call.parent_stmt, f)
-            follow_res = related_name_add_import_modules(follow_res, search)
+            follow_res, search = evaluate.goto(call.parent, f)
+            follow_res = usages_add_import_modules(follow_res, search)
 
-            #print follow_res, [d.parent for d in follow_res]
+            compare_follow_res = compare_array(follow_res)
             # compare to see if they match
-            if any(r in definitions for r in follow_res):
-                scope = call.parent_stmt
-                result.append(api_classes.RelatedName(search, scope))
+            if any(r in compare_definitions for r in compare_follow_res):
+                scope = call.parent
+                result.append(api_classes.Usage(search, scope))
 
         return result
 
     if not definitions:
         return set()
 
-    def is_definition(arr):
-        try:
-            for a in arr:
-                assert len(a) == 1
-                a = a[0]
-                if a.isinstance(parsing.Array):
-                    assert is_definition(a)
-                elif a.isinstance(parsing.Call):
-                    assert a.execution is None
-            return True
-        except AssertionError:
-            return False
-
+    compare_definitions = compare_array(definitions)
     mods |= set([d.get_parent_until() for d in definitions])
     names = []
     for m in get_directory_modules_for_name(mods, search_name):
@@ -413,7 +498,7 @@ def related_names(definitions, search_name, mods):
         except KeyError:
             continue
         for stmt in stmts:
-            if isinstance(stmt, parsing.Import):
+            if isinstance(stmt, pr.Import):
                 count = 0
                 imps = []
                 for i in stmt.get_all_import_names():
@@ -424,84 +509,82 @@ def related_names(definitions, search_name, mods):
 
                 for used_count, name_part in imps:
                     i = imports.ImportPath(stmt, kill_count=count - used_count,
-                                                        direct_resolve=True)
+                                           direct_resolve=True)
                     f = i.follow(is_goto=True)
                     if set(f) & set(definitions):
-                        names.append(api_classes.RelatedName(name_part, stmt))
+                        names.append(api_classes.Usage(name_part, stmt))
             else:
-                calls = _scan_array(stmt.get_assignment_calls(), search_name)
-                for d in stmt.assignment_details:
-                    if not is_definition(d[1]):
-                        calls += _scan_array(d[1], search_name)
-                for call in calls:
+                for call in _scan_statement(stmt, search_name,
+                                            assignment_details=True):
                     names += check_call(call)
     return names
 
 
-def related_name_add_import_modules(definitions, search_name):
+def usages_add_import_modules(definitions, search_name):
     """ Adds the modules of the imports """
     new = set()
     for d in definitions:
-        if isinstance(d.parent, parsing.Import):
+        if isinstance(d.parent, pr.Import):
             s = imports.ImportPath(d.parent, direct_resolve=True)
-            try:
+            with common.ignored(IndexError):
                 new.add(s.follow(is_goto=True)[0])
-            except IndexError:
-                pass
     return set(definitions) | new
 
 
 def check_flow_information(flow, search_name, pos):
     """ Try to find out the type of a variable just with the information that
-    is given by the flows: e.g. It is also responsible for assert checks.
-    >>> if isinstance(k, str):
-    >>>     k.  # <- completion here
+    is given by the flows: e.g. It is also responsible for assert checks.::
+
+        if isinstance(k, str):
+            k.  # <- completion here
 
     ensures that `k` is a string.
     """
+    if not settings.dynamic_flow_information:
+        return None
     result = []
-    if isinstance(flow, parsing.Scope) and not result:
+    if isinstance(flow, (pr.Scope, fast_parser.Module)) and not result:
         for ass in reversed(flow.asserts):
             if pos is None or ass.start_pos > pos:
                 continue
-            result = check_statement_information(ass, search_name)
+            result = _check_isinstance_type(ass, search_name)
             if result:
                 break
 
-    if isinstance(flow, parsing.Flow) and not result:
-        if flow.command in ['if', 'while'] and len(flow.inits) == 1:
-            result = check_statement_information(flow.inits[0], search_name)
+    if isinstance(flow, pr.Flow) and not result:
+        if flow.command in ['if', 'while'] and len(flow.inputs) == 1:
+            result = _check_isinstance_type(flow.inputs[0], search_name)
     return result
 
 
-def check_statement_information(stmt, search_name):
+def _check_isinstance_type(stmt, search_name):
     try:
-        ass = stmt.get_assignment_calls()
-        try:
-            call = ass.get_only_subelement()
-        except AttributeError:
-            assert False
-        assert type(call) == parsing.Call and str(call.name) == 'isinstance'
+        commands = stmt.get_commands()
+        # this might be removed if we analyze and, etc
+        assert len(commands) == 1
+        call = commands[0]
+        assert type(call) is pr.Call and str(call.name) == 'isinstance'
         assert bool(call.execution)
 
         # isinstance check
         isinst = call.execution.values
         assert len(isinst) == 2  # has two params
-        assert len(isinst[0]) == 1
-        assert len(isinst[1]) == 1
-        assert isinstance(isinst[0][0], parsing.Call)
+        obj, classes = [statement.get_commands() for statement in isinst]
+        assert len(obj) == 1
+        assert len(classes) == 1
+        assert isinstance(obj[0], pr.Call)
         # names fit?
-        assert str(isinst[0][0].name) == search_name
-        classes_call = isinst[1][0]  # class_or_type_or_tuple
-        assert isinstance(classes_call, parsing.Call)
-        result = []
-        for c in evaluate.follow_call(classes_call):
-            if isinstance(c, evaluate.Array):
-                result += c.get_index_types()
-            else:
-                result.append(c)
-        for i, c in enumerate(result):
-            result[i] = evaluate.Instance(c)
-        return result
+        assert str(obj[0].name) == search_name
+        assert isinstance(classes[0], pr.Call)  # can be type or tuple
     except AssertionError:
         return []
+
+    result = []
+    for c in evaluate.follow_call(classes[0]):
+        if isinstance(c, er.Array):
+            result += c.get_index_types()
+        else:
+            result.append(c)
+    for i, c in enumerate(result):
+        result[i] = er.Instance(c)
+    return result

jedi/evaluate_representation.py

@@ -0,0 +1,940 @@
+"""
+Like described in the :mod:`parsing_representation` module, there's a need for
+an ast like module to represent the states of parsed modules.
+
+But now there are also structures in Python that need a little bit more than
+that. An ``Instance`` for example is only a ``Class`` before it is
+instantiated. This class represents these cases.
+
+So, why is there also a ``Class`` class here? Well, there are decorators and
+they change classes in Python 3.
+"""
+from __future__ import with_statement
+
+import copy
+import itertools
+
+from jedi._compatibility import use_metaclass, next, hasattr, unicode
+from jedi import parsing_representation as pr
+from jedi import cache
+from jedi import helpers
+from jedi import debug
+from jedi import common
+import recursion
+import docstrings
+import imports
+import evaluate
+import builtin
+import dynamic
+
+
+class Executable(pr.IsScope):
+    """
+    An instance is also an executable - because __init__ is called
+    :param var_args: The param input array, consist of `pr.Array` or list.
+    """
+    def __init__(self, base, var_args=()):
+        self.base = base
+        self.var_args = var_args
+
+    def get_parent_until(self, *args, **kwargs):
+        return self.decorated.get_parent_until(*args, **kwargs)
+
+    @property
+    def parent(self):
+        return self.decorated.parent
+
+    @property
+    def decorated(self):
+        """
+        Instance doesn't care about decorators and Execution overrides this
+        """
+        return self.base
+
+
+class Instance(use_metaclass(cache.CachedMetaClass, Executable)):
+    """
+    This class is used to evaluate instances.
+    """
+    def __init__(self, base, var_args=()):
+        super(Instance, self).__init__(base, var_args)
+        if str(base.name) in ['list', 'set'] \
+                and builtin.Builtin.scope == base.get_parent_until():
+            # compare the module path with the builtin name.
+            self.var_args = dynamic.check_array_instances(self)
+        else:
+            # need to execute the __init__ function, because the dynamic param
+            # searching needs it.
+            with common.ignored(KeyError):
+                self.execute_subscope_by_name('__init__', self.var_args)
+        # Generated instances are classes that are just generated by self
+        # (No var_args) used.
+        self.is_generated = False
+
+    @cache.memoize_default()
+    def _get_method_execution(self, func):
+        func = InstanceElement(self, func, True)
+        return Execution(func, self.var_args)
+
+    def _get_func_self_name(self, func):
+        """
+        Returns the name of the first param in a class method (which is
+        normally self.
+        """
+        try:
+            return str(func.params[0].used_vars[0])
+        except IndexError:
+            return None
+
+    @cache.memoize_default([])
+    def _get_self_attributes(self):
+        def add_self_dot_name(name):
+            """
+            Need to copy and rewrite the name, because names are now
+            ``instance_usage.variable`` instead of ``self.variable``.
+            """
+            n = copy.copy(name)
+            n.names = n.names[1:]
+            names.append(InstanceElement(self, n))
+
+        names = []
+        # This loop adds the names of the self object, copies them and removes
+        # the self.
+        for sub in self.base.subscopes:
+            if isinstance(sub, pr.Class):
+                continue
+            # Get the self name, if there's one.
+            self_name = self._get_func_self_name(sub)
+            if not self_name:
+                continue
+
+            if sub.name.get_code() == '__init__':
+                # ``__init__`` is special because the params need are injected
+                # this way. Therefore an execution is necessary.
+                if not sub.decorators:
+                    # __init__ decorators should generally just be ignored,
+                    # because to follow them and their self variables is too
+                    # complicated.
+                    sub = self._get_method_execution(sub)
+            for n in sub.get_set_vars():
+                # Only names with the selfname are being added.
+                # It is also important, that they have a len() of 2,
+                # because otherwise, they are just something else
+                if n.names[0] == self_name and len(n.names) == 2:
+                    add_self_dot_name(n)
+
+        for s in self.base.get_super_classes():
+            names += Instance(s)._get_self_attributes()
+
+        return names
+
+    def get_subscope_by_name(self, name):
+        sub = self.base.get_subscope_by_name(name)
+        return InstanceElement(self, sub, True)
+
+    def execute_subscope_by_name(self, name, args=()):
+        method = self.get_subscope_by_name(name)
+        return Execution(method, args).get_return_types()
+
+    def get_descriptor_return(self, obj):
+        """ Throws a KeyError if there's no method. """
+        # Arguments in __get__ descriptors are obj, class.
+        # `method` is the new parent of the array, don't know if that's good.
+        args = [obj, obj.base] if isinstance(obj, Instance) else [None, obj]
+        return self.execute_subscope_by_name('__get__', args)
+
+    @cache.memoize_default([])
+    def get_defined_names(self):
+        """
+        Get the instance vars of a class. This includes the vars of all
+        classes
+        """
+        names = self._get_self_attributes()
+
+        class_names = self.base.get_defined_names()
+        for var in class_names:
+            names.append(InstanceElement(self, var, True))
+        return names
+
+    def scope_generator(self):
+        """
+        An Instance has two scopes: The scope with self names and the class
+        scope. Instance variables have priority over the class scope.
+        """
+        yield self, self._get_self_attributes()
+
+        names = []
+        class_names = self.base.get_defined_names()
+        for var in class_names:
+            names.append(InstanceElement(self, var, True))
+        yield self, names
+
+    def get_index_types(self, index=None):
+        args = [] if index is None else [index]
+        try:
+            return self.execute_subscope_by_name('__getitem__', args)
+        except KeyError:
+            debug.warning('No __getitem__, cannot access the array.')
+            return []
+
+    def __getattr__(self, name):
+        if name not in ['start_pos', 'end_pos', 'name', 'get_imports',
+                        'doc', 'docstr', 'asserts']:
+            raise AttributeError("Instance %s: Don't touch this (%s)!"
+                                 % (self, name))
+        return getattr(self.base, name)
+
+    def __repr__(self):
+        return "<e%s of %s (var_args: %s)>" % \
+            (type(self).__name__, self.base, len(self.var_args or []))
+
+
+class InstanceElement(use_metaclass(cache.CachedMetaClass, pr.Base)):
+    """
+    InstanceElement is a wrapper for any object, that is used as an instance
+    variable (e.g. self.variable or class methods).
+    """
+    def __init__(self, instance, var, is_class_var=False):
+        if isinstance(var, pr.Function):
+            var = Function(var)
+        elif isinstance(var, pr.Class):
+            var = Class(var)
+        self.instance = instance
+        self.var = var
+        self.is_class_var = is_class_var
+
+    @property
+    @cache.memoize_default()
+    def parent(self):
+        par = self.var.parent
+        if isinstance(par, Class) and par == self.instance.base \
+            or isinstance(par, pr.Class) \
+                and par == self.instance.base.base:
+            par = self.instance
+        elif not isinstance(par, pr.Module):
+            par = InstanceElement(self.instance, par, self.is_class_var)
+        return par
+
+    def get_parent_until(self, *args, **kwargs):
+        return pr.Simple.get_parent_until(self, *args, **kwargs)
+
+    def get_decorated_func(self):
+        """ Needed because the InstanceElement should not be stripped """
+        func = self.var.get_decorated_func(self.instance)
+        if func == self.var:
+            return self
+        return func
+
+    def get_commands(self):
+        # Copy and modify the array.
+        return [InstanceElement(self.instance, command, self.is_class_var)
+                if not isinstance(command, unicode) else command
+                for command in self.var.get_commands()]
+
+    def __iter__(self):
+        for el in self.var.__iter__():
+            yield InstanceElement(self.instance, el, self.is_class_var)
+
+    def __getattr__(self, name):
+        return getattr(self.var, name)
+
+    def isinstance(self, *cls):
+        return isinstance(self.var, cls)
+
+    def __repr__(self):
+        return "<%s of %s>" % (type(self).__name__, self.var)
+
+
+class Class(use_metaclass(cache.CachedMetaClass, pr.IsScope)):
+    """
+    This class is not only important to extend `pr.Class`, it is also a
+    important for descriptors (if the descriptor methods are evaluated or not).
+    """
+    def __init__(self, base):
+        self.base = base
+
+    @cache.memoize_default(default=())
+    def get_super_classes(self):
+        supers = []
+        # TODO care for mro stuff (multiple super classes).
+        for s in self.base.supers:
+            # Super classes are statements.
+            for cls in evaluate.follow_statement(s):
+                if not isinstance(cls, Class):
+                    debug.warning('Received non class, as a super class')
+                    continue  # Just ignore other stuff (user input error).
+                supers.append(cls)
+        if not supers and self.base.parent != builtin.Builtin.scope:
+            # add `object` to classes
+            supers += evaluate.find_name(builtin.Builtin.scope, 'object')
+        return supers
+
+    @cache.memoize_default(default=())
+    def get_defined_names(self):
+        def in_iterable(name, iterable):
+            """ checks if the name is in the variable 'iterable'. """
+            for i in iterable:
+                # Only the last name is important, because these names have a
+                # maximal length of 2, with the first one being `self`.
+                if i.names[-1] == name.names[-1]:
+                    return True
+            return False
+
+        result = self.base.get_defined_names()
+        super_result = []
+        # TODO mro!
+        for cls in self.get_super_classes():
+            # Get the inherited names.
+            for i in cls.get_defined_names():
+                if not in_iterable(i, result):
+                    super_result.append(i)
+        result += super_result
+        return result
+
+    def get_subscope_by_name(self, name):
+        for sub in reversed(self.subscopes):
+            if sub.name.get_code() == name:
+                return sub
+        raise KeyError("Couldn't find subscope.")
+
+    @property
+    def name(self):
+        return self.base.name
+
+    def __getattr__(self, name):
+        if name not in ['start_pos', 'end_pos', 'parent', 'asserts', 'docstr',
+                        'doc', 'get_imports', 'get_parent_until', 'get_code',
+                        'subscopes']:
+            raise AttributeError("Don't touch this: %s of %s !" % (name, self))
+        return getattr(self.base, name)
+
+    def __repr__(self):
+        return "<e%s of %s>" % (type(self).__name__, self.base)
+
+
+class Function(use_metaclass(cache.CachedMetaClass, pr.IsScope)):
+    """
+    Needed because of decorators. Decorators are evaluated here.
+    """
+    def __init__(self, func, is_decorated=False):
+        """ This should not be called directly """
+        self.base_func = func
+        self.is_decorated = is_decorated
+
+    @cache.memoize_default()
+    def _decorated_func(self, instance=None):
+        """
+        Returns the function, that is to be executed in the end.
+        This is also the places where the decorators are processed.
+        """
+        f = self.base_func
+
+        # Only enter it, if has not already been processed.
+        if not self.is_decorated:
+            for dec in reversed(self.base_func.decorators):
+                debug.dbg('decorator:', dec, f)
+                dec_results = set(evaluate.follow_statement(dec))
+                if not len(dec_results):
+                    debug.warning('decorator not found: %s on %s' %
+                                 (dec, self.base_func))
+                    return None
+                decorator = dec_results.pop()
+                if dec_results:
+                    debug.warning('multiple decorators found', self.base_func,
+                                  dec_results)
+                # Create param array.
+                old_func = Function(f, is_decorated=True)
+                if instance is not None and decorator.isinstance(Function):
+                    old_func = InstanceElement(instance, old_func)
+                    instance = None
+
+                wrappers = Execution(decorator, (old_func,)).get_return_types()
+                if not len(wrappers):
+                    debug.warning('no wrappers found', self.base_func)
+                    return None
+                if len(wrappers) > 1:
+                    debug.warning('multiple wrappers found', self.base_func,
+                                  wrappers)
+                # This is here, that the wrapper gets executed.
+                f = wrappers[0]
+
+                debug.dbg('decorator end', f)
+        if f != self.base_func and isinstance(f, pr.Function):
+            f = Function(f)
+        return f
+
+    def get_decorated_func(self, instance=None):
+        decorated_func = self._decorated_func(instance)
+        if decorated_func == self.base_func:
+            return self
+        if decorated_func is None:
+            # If the decorator func is not found, just ignore the decorator
+            # function, because sometimes decorators are just really
+            # complicated.
+            return Function(self.base_func, True)
+        return decorated_func
+
+    def get_magic_method_names(self):
+        return builtin.Builtin.magic_function_scope.get_defined_names()
+
+    def get_magic_method_scope(self):
+        return builtin.Builtin.magic_function_scope
+
+    def __getattr__(self, name):
+        return getattr(self.base_func, name)
+
+    def __repr__(self):
+        dec = ''
+        if self._decorated_func() != self.base_func:
+            dec = " is " + repr(self._decorated_func())
+        return "<e%s of %s%s>" % (type(self).__name__, self.base_func, dec)
+
+
+class Execution(Executable):
+    """
+    This class is used to evaluate functions and their returns.
+
+    This is the most complicated class, because it contains the logic to
+    transfer parameters. It is even more complicated, because there may be
+    multiple calls to functions and recursion has to be avoided. But this is
+    responsibility of the decorators.
+    """
+    def follow_var_arg(self, index):
+        try:
+            stmt = self.var_args[index]
+        except IndexError:
+            return []
+        else:
+            if isinstance(stmt, pr.Statement):
+                return evaluate.follow_statement(stmt)
+            else:
+                return [stmt]  # just some arbitrary object
+
+    @property
+    @cache.memoize_default()
+    def decorated(self):
+        """Get the decorated version of the input"""
+        base = self.base
+        if self.base.isinstance(Function):
+            base = base.get_decorated_func()
+        return base
+
+    @cache.memoize_default(default=())
+    @recursion.ExecutionRecursionDecorator
+    def get_return_types(self, evaluate_generator=False):
+        """ Get the return types of a function. """
+        base = self.decorated
+        stmts = []
+        if base.parent == builtin.Builtin.scope \
+                and not isinstance(base, (Generator, Array)):
+            func_name = str(base.name)
+
+            # some implementations of builtins:
+            if func_name == 'getattr':
+                # follow the first param
+                objects = self.follow_var_arg(0)
+                names = self.follow_var_arg(1)
+                for obj in objects:
+                    if not isinstance(obj, (Instance, Class, pr.Module)):
+                        debug.warning('getattr called without instance')
+                        continue
+
+                    for arr_name in names:
+                        if not isinstance(arr_name, Instance):
+                            debug.warning('getattr called without str')
+                            continue
+                        if len(arr_name.var_args) != 1:
+                            debug.warning('jedi getattr is too simple')
+                        key = arr_name.var_args[0]
+                        stmts += evaluate.follow_path(iter([key]), obj, base)
+                return stmts
+            elif func_name == 'type':
+                # otherwise it would be a metaclass
+                if len(self.var_args) == 1:
+                    objects = self.follow_var_arg(0)
+                    return [o.base for o in objects if isinstance(o, Instance)]
+            elif func_name == 'super':
+                # TODO make this able to detect multiple inheritance supers
+                accept = (pr.Function,)
+                func = self.var_args.get_parent_until(accept)
+                if func.isinstance(*accept):
+                    cls = func.get_parent_until(accept + (pr.Class,),
+                                                include_current=False)
+                    if isinstance(cls, pr.Class):
+                        cls = Class(cls)
+                        su = cls.get_super_classes()
+                        if su:
+                            return [Instance(su[0])]
+                return []
+
+        if base.isinstance(Class):
+            # There maybe executions of executions.
+            stmts = [Instance(base, self.var_args)]
+        elif isinstance(base, Generator):
+            return base.iter_content()
+        else:
+            try:
+                base.returns  # Test if it is a function
+            except AttributeError:
+                if hasattr(base, 'execute_subscope_by_name'):
+                    try:
+                        stmts = base.execute_subscope_by_name('__call__',
+                                                              self.var_args)
+                    except KeyError:
+                        debug.warning("no __call__ func available", base)
+                else:
+                    debug.warning("no execution possible", base)
+            else:
+                stmts = self._get_function_returns(base, evaluate_generator)
+
+        debug.dbg('exec result: %s in %s' % (stmts, self))
+
+        return imports.strip_imports(stmts)
+
+    def _get_function_returns(self, func, evaluate_generator):
+        """ A normal Function execution """
+        # Feed the listeners, with the params.
+        for listener in func.listeners:
+            listener.execute(self.get_params())
+        if func.is_generator and not evaluate_generator:
+            return [Generator(func, self.var_args)]
+        else:
+            stmts = docstrings.find_return_types(func)
+            for r in self.returns:
+                if r is not None:
+                    stmts += evaluate.follow_statement(r)
+            return stmts
+
+    @cache.memoize_default(default=())
+    def get_params(self):
+        """
+        This returns the params for an Execution/Instance and is injected as a
+        'hack' into the pr.Function class.
+        This needs to be here, because Instance can have __init__ functions,
+        which act the same way as normal functions.
+        """
+        def gen_param_name_copy(param, keys=(), values=(), array_type=None):
+            """
+            Create a param with the original scope (of varargs) as parent.
+            """
+            if isinstance(self.var_args, pr.Array):
+                parent = self.var_args.parent
+                start_pos = self.var_args.start_pos
+            else:
+                parent = self.decorated
+                start_pos = 0, 0
+
+            new_param = copy.copy(param)
+            new_param.is_generated = True
+            if parent is not None:
+                new_param.parent = parent
+
+            # create an Array (-> needed for *args/**kwargs tuples/dicts)
+            arr = pr.Array(self._sub_module, start_pos, array_type, parent)
+            arr.values = values
+            key_stmts = []
+            for key in keys:
+                stmt = pr.Statement(self._sub_module, [], [], [],
+                                    start_pos, None)
+                stmt._commands = [key]
+                key_stmts.append(stmt)
+            arr.keys = key_stmts
+            arr.type = array_type
+
+            new_param._commands = [arr]
+
+            name = copy.copy(param.get_name())
+            name.parent = new_param
+            return name
+
+        result = []
+        start_offset = 0
+        if isinstance(self.decorated, InstanceElement):
+            # Care for self -> just exclude it and add the instance
+            start_offset = 1
+            self_name = copy.copy(self.decorated.params[0].get_name())
+            self_name.parent = self.decorated.instance
+            result.append(self_name)
+
+        param_dict = {}
+        for param in self.decorated.params:
+            param_dict[str(param.get_name())] = param
+        # There may be calls, which don't fit all the params, this just ignores
+        # it.
+        var_arg_iterator = self.get_var_args_iterator()
+
+        non_matching_keys = []
+        keys_used = set()
+        keys_only = False
+        for param in self.decorated.params[start_offset:]:
+            # The value and key can both be null. There, the defaults apply.
+            # args / kwargs will just be empty arrays / dicts, respectively.
+            # Wrong value count is just ignored. If you try to test cases that
+            # are not allowed in Python, Jedi will maybe not show any
+            # completions.
+            key, value = next(var_arg_iterator, (None, None))
+            while key:
+                keys_only = True
+                try:
+                    key_param = param_dict[str(key)]
+                except KeyError:
+                    non_matching_keys.append((key, value))
+                else:
+                    keys_used.add(str(key))
+                    result.append(gen_param_name_copy(key_param,
+                                                      values=[value]))
+                key, value = next(var_arg_iterator, (None, None))
+
+            commands = param.get_commands()
+            keys = []
+            values = []
+            array_type = None
+            ignore_creation = False
+            if commands[0] == '*':
+                # *args param
+                array_type = pr.Array.TUPLE
+                if value:
+                    values.append(value)
+                for key, value in var_arg_iterator:
+                    # Iterate until a key argument is found.
+                    if key:
+                        var_arg_iterator.push_back((key, value))
+                        break
+                    values.append(value)
+            elif commands[0] == '**':
+                # **kwargs param
+                array_type = pr.Array.DICT
+                if non_matching_keys:
+                    keys, values = zip(*non_matching_keys)
+            elif not keys_only:
+                # normal param
+                if value is not None:
+                    values = [value]
+                else:
+                    if param.assignment_details:
+                        # No value: return the default values.
+                        ignore_creation = True
+                        result.append(param.get_name())
+                        param.is_generated = True
+                    else:
+                        # If there is no assignment detail, that means there is
+                        # no assignment, just the result. Therefore nothing has
+                        # to be returned.
+                        values = []
+
+            # Just ignore all the params that are without a key, after one
+            # keyword argument was set.
+            if not ignore_creation and (not keys_only or commands[0] == '**'):
+                keys_used.add(str(key))
+                result.append(gen_param_name_copy(param, keys=keys,
+                                                  values=values, array_type=array_type))
+
+        if keys_only:
+            # sometimes param arguments are not completely written (which would
+            # create an Exception, but we have to handle that).
+            for k in set(param_dict) - keys_used:
+                result.append(gen_param_name_copy(param_dict[k]))
+        return result
+
+    def get_var_args_iterator(self):
+        """
+        Yields a key/value pair, the key is None, if its not a named arg.
+        """
+        def iterate():
+            # `var_args` is typically an Array, and not a list.
+            for stmt in self.var_args:
+                if not isinstance(stmt, pr.Statement):
+                    if stmt is None:
+                        yield None, None
+                        continue
+                    old = stmt
+                    # generate a statement if it's not already one.
+                    module = builtin.Builtin.scope
+                    stmt = pr.Statement(module, [], [], [], (0, 0), None)
+                    stmt._commands = [old]
+
+                # *args
+                commands = stmt.get_commands()
+                if not len(commands):
+                    continue
+                if commands[0] == '*':
+                    arrays = evaluate.follow_call_list(commands[1:])
+                    # *args must be some sort of an array, otherwise -> ignore
+
+                    for array in arrays:
+                        if isinstance(array, Array):
+                            for field_stmt in array:  # yield from plz!
+                                yield None, field_stmt
+                        elif isinstance(array, Generator):
+                            for field_stmt in array.iter_content():
+                                yield None, helpers.FakeStatement(field_stmt)
+                # **kwargs
+                elif commands[0] == '**':
+                    arrays = evaluate.follow_call_list(commands[1:])
+                    for array in arrays:
+                        if isinstance(array, Array):
+                            for key_stmt, value_stmt in array.items():
+                                # first index, is the key if syntactically correct
+                                call = key_stmt.get_commands()[0]
+                                if isinstance(call, pr.Name):
+                                    yield call, value_stmt
+                                elif type(call) is pr.Call:
+                                    yield call.name, value_stmt
+                # Normal arguments (including key arguments).
+                else:
+                    if stmt.assignment_details:
+                        key_arr, op = stmt.assignment_details[0]
+                        # named parameter
+                        if key_arr and isinstance(key_arr[0], pr.Call):
+                            yield key_arr[0].name, stmt
+                    else:
+                        yield None, stmt
+
+        return iter(common.PushBackIterator(iterate()))
+
+    def get_defined_names(self):
+        """
+        Call the default method with the own instance (self implements all
+        the necessary functions). Add also the params.
+        """
+        return self.get_params() + pr.Scope.get_set_vars(self)
+
+    get_set_vars = get_defined_names
+
+    @common.rethrow_uncaught
+    def copy_properties(self, prop):
+        """
+        Literally copies a property of a Function. Copying is very expensive,
+        because it is something like `copy.deepcopy`. However, these copied
+        objects can be used for the executions, as if they were in the
+        execution.
+        """
+        # Copy all these lists into this local function.
+        attr = getattr(self.decorated, prop)
+        objects = []
+        for element in attr:
+            if element is None:
+                copied = element
+            else:
+                copied = helpers.fast_parent_copy(element)
+                copied.parent = self._scope_copy(copied.parent)
+                if isinstance(copied, pr.Function):
+                    copied = Function(copied)
+            objects.append(copied)
+        return objects
+
+    def __getattr__(self, name):
+        if name not in ['start_pos', 'end_pos', 'imports', '_sub_module']:
+            raise AttributeError('Tried to access %s: %s. Why?' % (name, self))
+        return getattr(self.decorated, name)
+
+    @cache.memoize_default()
+    @common.rethrow_uncaught
+    def _scope_copy(self, scope):
+        """ Copies a scope (e.g. if) in an execution """
+        # TODO method uses different scopes than the subscopes property.
+
+        # just check the start_pos, sometimes it's difficult with closures
+        # to compare the scopes directly.
+        if scope.start_pos == self.start_pos:
+            return self
+        else:
+            copied = helpers.fast_parent_copy(scope)
+            copied.parent = self._scope_copy(copied.parent)
+            return copied
+
+    @property
+    @cache.memoize_default()
+    def returns(self):
+        return self.copy_properties('returns')
+
+    @property
+    @cache.memoize_default()
+    def asserts(self):
+        return self.copy_properties('asserts')
+
+    @property
+    @cache.memoize_default()
+    def statements(self):
+        return self.copy_properties('statements')
+
+    @property
+    @cache.memoize_default()
+    def subscopes(self):
+        return self.copy_properties('subscopes')
+
+    def get_statement_for_position(self, pos):
+        return pr.Scope.get_statement_for_position(self, pos)
+
+    def __repr__(self):
+        return "<%s of %s>" % \
+            (type(self).__name__, self.decorated)
+
+
+class Generator(use_metaclass(cache.CachedMetaClass, pr.Base)):
+    """ Cares for `yield` statements. """
+    def __init__(self, func, var_args):
+        super(Generator, self).__init__()
+        self.func = func
+        self.var_args = var_args
+
+    def get_defined_names(self):
+        """
+        Returns a list of names that define a generator, which can return the
+        content of a generator.
+        """
+        names = []
+        none_pos = (0, 0)
+        executes_generator = ('__next__', 'send')
+        for n in ('close', 'throw') + executes_generator:
+            name = pr.Name(builtin.Builtin.scope, [(n, none_pos)],
+                           none_pos, none_pos)
+            if n in executes_generator:
+                name.parent = self
+            else:
+                name.parent = builtin.Builtin.scope
+            names.append(name)
+        debug.dbg('generator names', names)
+        return names
+
+    def iter_content(self):
+        """ returns the content of __iter__ """
+        return Execution(self.func, self.var_args).get_return_types(True)
+
+    def get_index_types(self, index=None):
+        debug.warning('Tried to get array access on a generator', self)
+        return []
+
+    def __getattr__(self, name):
+        if name not in ['start_pos', 'end_pos', 'parent', 'get_imports',
+                        'asserts', 'doc', 'docstr', 'get_parent_until', 'get_code',
+                        'subscopes']:
+            raise AttributeError("Accessing %s of %s is not allowed."
+                                 % (self, name))
+        return getattr(self.func, name)
+
+    def __repr__(self):
+        return "<%s of %s>" % (type(self).__name__, self.func)
+
+
+class Array(use_metaclass(cache.CachedMetaClass, pr.Base)):
+    """
+    Used as a mirror to pr.Array, if needed. It defines some getter
+    methods which are important in this module.
+    """
+    def __init__(self, array):
+        self._array = array
+
+    def get_index_types(self, index_arr=None):
+        """ Get the types of a specific index or all, if not given """
+        if index_arr is not None:
+            if index_arr and [x for x in index_arr if ':' in x.get_commands()]:
+                # array slicing
+                return [self]
+
+            index_possibilities = self._follow_values(index_arr)
+            if len(index_possibilities) == 1:
+                # This is indexing only one element, with a fixed index number,
+                # otherwise it just ignores the index (e.g. [1+1]).
+                index = index_possibilities[0]
+                if isinstance(index, Instance) \
+                        and str(index.name) in ['int', 'str'] \
+                        and len(index.var_args) == 1:
+                    # TODO this is just very hackish and a lot of use cases are
+                    # being ignored
+                    with common.ignored(KeyError, IndexError,
+                                        UnboundLocalError, TypeError):
+                        return self.get_exact_index_types(index.var_args[0])
+
+        result = list(self._follow_values(self._array.values))
+        result += dynamic.check_array_additions(self)
+        return set(result)
+
+    def get_exact_index_types(self, mixed_index):
+        """ Here the index is an int/str. Raises IndexError/KeyError """
+        index = mixed_index
+        if self.type == pr.Array.DICT:
+            index = None
+            for i, key_statement in enumerate(self._array.keys):
+                # Because we only want the key to be a string.
+                key_commands = key_statement.get_commands()
+                if len(key_commands) != 1:  # cannot deal with complex strings
+                    continue
+                key = key_commands[0]
+                if isinstance(key, pr.Call) and key.type == pr.Call.STRING:
+                    str_key = key.name
+                elif isinstance(key, pr.Name):
+                    str_key = str(key)
+
+                if mixed_index == str_key:
+                    index = i
+                    break
+            if index is None:
+                raise KeyError('No key found in dictionary')
+
+        # Can raise an IndexError
+        values = [self._array.values[index]]
+        return self._follow_values(values)
+
+    def _follow_values(self, values):
+        """ helper function for the index getters """
+        return list(itertools.chain.from_iterable(evaluate.follow_statement(v)
+                                                  for v in values))
+
+    def get_defined_names(self):
+        """
+        This method generates all `ArrayMethod` for one pr.Array.
+        It returns e.g. for a list: append, pop, ...
+        """
+        # `array.type` is a string with the type, e.g. 'list'.
+        scope = evaluate.find_name(builtin.Builtin.scope, self._array.type)[0]
+        scope = Instance(scope)
+        names = scope.get_defined_names()
+        return [ArrayMethod(n) for n in names]
+
+    @property
+    def parent(self):
+        return builtin.Builtin.scope
+
+    def get_parent_until(self):
+        return builtin.Builtin.scope
+
+    def __getattr__(self, name):
+        if name not in ['type', 'start_pos', 'get_only_subelement', 'parent',
+                        'get_parent_until', 'items']:
+            raise AttributeError('Strange access on %s: %s.' % (self, name))
+        return getattr(self._array, name)
+
+    def __getitem__(self):
+        return self._array.__getitem__()
+
+    def __iter__(self):
+        return self._array.__iter__()
+
+    def __len__(self):
+        return self._array.__len__()
+
+    def __repr__(self):
+        return "<e%s of %s>" % (type(self).__name__, self._array)
+
+
+class ArrayMethod(object):
+    """
+    A name, e.g. `list.append`, it is used to access the original array
+    methods.
+    """
+    def __init__(self, name):
+        super(ArrayMethod, self).__init__()
+        self.name = name
+
+    def __getattr__(self, name):
+        # Set access privileges:
+        if name not in ['parent', 'names', 'start_pos', 'end_pos', 'get_code']:
+            raise AttributeError('Strange accesson %s: %s.' % (self, name))
+        return getattr(self.name, name)
+
+    def get_parent_until(self):
+        return builtin.Builtin.scope
+
+    def __repr__(self):
+        return "<%s of %s>" % (type(self).__name__, self.name)

jedi/fast_parser.py

@@ -0,0 +1,435 @@
+"""
+Basically a parser that is faster, because it tries to parse only parts and if
+anything changes, it only reparses the changed parts. But because it's not
+finished (and still not working as I want), I won't document it any further.
+"""
+import re
+
+from jedi._compatibility import use_metaclass
+from jedi import settings
+from jedi import parsing
+from jedi import parsing_representation as pr
+from jedi import cache
+from jedi import common
+
+
+SCOPE_CONTENTS = ['asserts', 'subscopes', 'imports', 'statements', 'returns']
+
+
+class Module(pr.Simple, pr.Module):
+    def __init__(self, parsers):
+        super(Module, self).__init__(self, (1, 0))
+        self.parsers = parsers
+        self.reset_caches()
+
+        self.start_pos = 1, 0
+        self.end_pos = None, None
+
+    def reset_caches(self):
+        """ This module does a whole lot of caching, because it uses different
+        parsers. """
+        self._used_names = None
+        for p in self.parsers:
+            p.user_scope = None
+            p.user_stmt = None
+
+    def __getattr__(self, name):
+        if name.startswith('__'):
+            raise AttributeError('Not available!')
+        else:
+            return getattr(self.parsers[0].module, name)
+
+    @property
+    def used_names(self):
+        if self._used_names is None:
+            dct = {}
+            for p in self.parsers:
+                for k, statement_set in p.module.used_names.items():
+                    if k in dct:
+                        dct[k] |= statement_set
+                    else:
+                        dct[k] = set(statement_set)
+
+            self._used_names = dct
+        return self._used_names
+
+    def __repr__(self):
+        return "<%s: %s@%s-%s>" % (type(self).__name__, self.name,
+                                   self.start_pos[0], self.end_pos[0])
+
+
+class CachedFastParser(type):
+    """ This is a metaclass for caching `FastParser`. """
+    def __call__(self, source, module_path=None, user_position=None):
+        if not settings.fast_parser:
+            return parsing.Parser(source, module_path, user_position)
+
+        pi = cache.parser_cache.get(module_path, None)
+        if pi is None or isinstance(pi.parser, parsing.Parser):
+            p = super(CachedFastParser, self).__call__(source, module_path,
+                                                       user_position)
+        else:
+            p = pi.parser  # pi is a `cache.ParserCacheItem`
+            p.update(source, user_position)
+        return p
+
+
+class ParserNode(object):
+    def __init__(self, parser, code, parent=None):
+        self.parent = parent
+        self.code = code
+        self.hash = hash(code)
+
+        self.children = []
+        # must be created before new things are added to it.
+        self.save_contents(parser)
+
+    def save_contents(self, parser):
+        self.parser = parser
+
+        try:
+            # with fast_parser we have either 1 subscope or only statements.
+            self.content_scope = parser.module.subscopes[0]
+        except IndexError:
+            self.content_scope = parser.module
+
+        scope = self.content_scope
+        self._contents = {}
+        for c in SCOPE_CONTENTS:
+            self._contents[c] = list(getattr(scope, c))
+        self._is_generator = scope.is_generator
+
+        self.old_children = self.children
+        self.children = []
+
+    def reset_contents(self):
+        scope = self.content_scope
+        for key, c in self._contents.items():
+            setattr(scope, key, list(c))
+        scope.is_generator = self._is_generator
+        self.parser.user_scope = self.parser.module
+
+        if self.parent is None:
+            # Global vars of the first one can be deleted, in the global scope
+            # they make no sense.
+            self.parser.module.global_vars = []
+
+        for c in self.children:
+            c.reset_contents()
+
+    def parent_until_indent(self, indent=None):
+        if indent is None or self.indent >= indent and self.parent:
+            self.old_children = []
+            if self.parent is not None:
+                return self.parent.parent_until_indent(indent)
+        return self
+
+    @property
+    def indent(self):
+        if not self.parent:
+            return 0
+        module = self.parser.module
+        try:
+            el = module.subscopes[0]
+        except IndexError:
+            try:
+                el = module.statements[0]
+            except IndexError:
+                try:
+                    el = module.imports[0]
+                except IndexError:
+                    try:
+                        el = [r for r in module.returns if r is not None][0]
+                    except IndexError:
+                        return self.parent.indent + 1
+        return el.start_pos[1]
+
+    def _set_items(self, parser, set_parent=False):
+        # insert parser objects into current structure
+        scope = self.content_scope
+        for c in SCOPE_CONTENTS:
+            content = getattr(scope, c)
+            items = getattr(parser.module, c)
+            if set_parent:
+                for i in items:
+                    if i is None:
+                        continue  # happens with empty returns
+                    i.parent = scope.use_as_parent
+                    if isinstance(i, (pr.Function, pr.Class)):
+                        for d in i.decorators:
+                            d.parent = scope.use_as_parent
+            content += items
+
+        # global_vars
+        cur = self
+        while cur.parent is not None:
+            cur = cur.parent
+        cur.parser.module.global_vars += parser.module.global_vars
+
+        scope.is_generator |= parser.module.is_generator
+
+    def add_node(self, node, set_parent=False):
+        """Adding a node means adding a node that was already added earlier"""
+        self.children.append(node)
+        self._set_items(node.parser, set_parent=set_parent)
+        node.old_children = node.children
+        node.children = []
+        return node
+
+    def add_parser(self, parser, code):
+        return self.add_node(ParserNode(parser, code, self), True)
+
+
+class FastParser(use_metaclass(CachedFastParser)):
+    def __init__(self, code, module_path=None, user_position=None):
+        # set values like `pr.Module`.
+        self.module_path = module_path
+        self.user_position = user_position
+        self._user_scope = None
+
+        self.current_node = None
+        self.parsers = []
+        self.module = Module(self.parsers)
+        self.reset_caches()
+
+        try:
+            self._parse(code)
+        except:
+            # FastParser is cached, be careful with exceptions
+            self.parsers[:] = []
+            raise
+
+    @property
+    def user_scope(self):
+        if self._user_scope is None:
+            for p in self.parsers:
+                if p.user_scope:
+                    if isinstance(p.user_scope, pr.SubModule):
+                        continue
+                    self._user_scope = p.user_scope
+
+        if isinstance(self._user_scope, pr.SubModule) \
+                or self._user_scope is None:
+            self._user_scope = self.module
+        return self._user_scope
+
+    @property
+    def user_stmt(self):
+        if self._user_stmt is None:
+            for p in self.parsers:
+                if p.user_stmt:
+                    self._user_stmt = p.user_stmt
+                    break
+        return self._user_stmt
+
+    def update(self, code, user_position=None):
+        self.user_position = user_position
+        self.reset_caches()
+
+
+        try:
+            self._parse(code)
+        except:
+            # FastParser is cached, be careful with exceptions
+            self.parsers[:] = []
+            raise
+
+    def _scan_user_scope(self, sub_module):
+        """ Scan with self.user_position. """
+        for scope in sub_module.statements + sub_module.subscopes:
+            if isinstance(scope, pr.Scope):
+                if scope.start_pos <= self.user_position <= scope.end_pos:
+                    return self._scan_user_scope(scope) or scope
+        return None
+
+    def _split_parts(self, code):
+        """
+        Split the code into different parts. This makes it possible to parse
+        each part seperately and therefore cache parts of the file and not
+        everything.
+        """
+        def add_part():
+            txt = '\n'.join(current_lines)
+            if txt:
+                if add_to_last and parts:
+                    parts[-1] += '\n' + txt
+                else:
+                    parts.append(txt)
+                current_lines[:] = []
+
+        r_keyword = '^[ \t]*(def|class|@|%s)' % '|'.join(common.FLOWS)
+
+        lines = code.splitlines()
+        current_lines = []
+        parts = []
+        is_decorator = False
+        current_indent = 0
+        old_indent = 0
+        new_indent = False
+        in_flow = False
+        add_to_last = False
+        # All things within flows are simply being ignored.
+        for i, l in enumerate(lines):
+            # check for dedents
+            m = re.match('^([\t ]*)(.?)', l)
+            indent = len(m.group(1))
+            if m.group(2) in ['', '#']:
+                current_lines.append(l)  # just ignore comments and blank lines
+                continue
+
+            if indent < current_indent:  # -> dedent
+                current_indent = indent
+                new_indent = False
+                if not in_flow or indent < old_indent:
+                    add_part()
+                    add_to_last = False
+                in_flow = False
+            elif new_indent:
+                current_indent = indent
+                new_indent = False
+
+            # Check lines for functions/classes and split the code there.
+            if not in_flow:
+                m = re.match(r_keyword, l)
+                if m:
+                    in_flow = m.group(1) in common.FLOWS
+                    if not is_decorator and not in_flow:
+                        add_part()
+                        add_to_last = False
+                    is_decorator = '@' == m.group(1)
+                    if not is_decorator:
+                        old_indent = current_indent
+                        current_indent += 1  # it must be higher
+                        new_indent = True
+                elif is_decorator:
+                    is_decorator = False
+                    add_to_last = True
+
+            current_lines.append(l)
+        add_part()
+
+        return parts
+
+    def _parse(self, code):
+        """ :type code: str """
+        def empty_parser():
+            new, temp = self._get_parser('', '', 0, [], False)
+            return new
+
+        parts = self._split_parts(code)
+        self.parsers[:] = []
+
+        line_offset = 0
+        start = 0
+        p = None
+        is_first = True
+
+        for code_part in parts:
+            lines = code_part.count('\n') + 1
+            if is_first or line_offset >= p.end_pos[0]:
+                indent = len(re.match(r'[ \t]*', code_part).group(0))
+                if is_first and self.current_node is not None:
+                    nodes = [self.current_node]
+                else:
+                    nodes = []
+                if self.current_node is not None:
+
+                    self.current_node = \
+                        self.current_node.parent_until_indent(indent)
+                    nodes += self.current_node.old_children
+
+                # check if code_part has already been parsed
+                # print '#'*45,line_offset, p and p.end_pos, '\n', code_part
+                p, node = self._get_parser(code_part, code[start:],
+                                           line_offset, nodes, not is_first)
+
+                if is_first and p.module.subscopes:
+                    # special case, we cannot use a function subscope as a
+                    # base scope, subscopes would save all the other contents
+                    new = empty_parser()
+                    if self.current_node is None:
+                        self.current_node = ParserNode(new, '')
+                    else:
+                        self.current_node.save_contents(new)
+                    self.parsers.append(new)
+                    is_first = False
+
+                if is_first:
+                    if self.current_node is None:
+                        self.current_node = ParserNode(p, code_part)
+                    else:
+                        self.current_node.save_contents(p)
+                else:
+                    if node is None:
+                        self.current_node = \
+                            self.current_node.add_parser(p, code_part)
+                    else:
+                        self.current_node = self.current_node.add_node(node)
+
+                if self.current_node.parent and (isinstance(p.user_scope,
+                                                            pr.SubModule) or p.user_scope is None) \
+                        and self.user_position \
+                        and p.start_pos <= self.user_position < p.end_pos:
+                    p.user_scope = self.current_node.parent.content_scope
+
+                self.parsers.append(p)
+
+                is_first = False
+            else:
+                # print '#'*45, line_offset, p.end_pos, 'theheck\n', code_part
+                pass
+
+            line_offset += lines
+            start += len(code_part) + 1  # +1 for newline
+
+        if self.parsers:
+            self.current_node = self.current_node.parent_until_indent()
+        else:
+            self.parsers.append(empty_parser())
+
+        self.module.end_pos = self.parsers[-1].end_pos
+
+        # print(self.parsers[0].module.get_code())
+        del code
+
+    def _get_parser(self, code, parser_code, line_offset, nodes, no_docstr):
+        h = hash(code)
+        hashes = [n.hash for n in nodes]
+        node = None
+        try:
+            index = hashes.index(h)
+            if nodes[index].code != code:
+                raise ValueError()
+        except ValueError:
+            p = parsing.Parser(parser_code, self.module_path,
+                               self.user_position, offset=(line_offset, 0),
+                               is_fast_parser=True, top_module=self.module,
+                               no_docstr=no_docstr)
+            p.module.parent = self.module
+        else:
+            if nodes[index] != self.current_node:
+                offset = int(nodes[0] == self.current_node)
+                self.current_node.old_children.pop(index - offset)
+            node = nodes.pop(index)
+            p = node.parser
+            m = p.module
+            m.line_offset += line_offset + 1 - m.start_pos[0]
+            if self.user_position is not None and \
+                    m.start_pos[0] <= self.user_position[0] <= m.end_pos[0]:
+                # It's important to take care of the whole user
+                # positioning stuff, if no reparsing is being done.
+                p.user_stmt = m.get_statement_for_position(
+                    self.user_position, include_imports=True)
+                if p.user_stmt:
+                    p.user_scope = p.user_stmt.parent
+                else:
+                    p.user_scope = self._scan_user_scope(m) or m
+
+        return p, node
+
+    def reset_caches(self):
+        self._user_scope = None
+        self._user_stmt = None
+        self.module.reset_caches()
+        if self.current_node is not None:
+            self.current_node.reset_contents()

jedi/helpers.py

@@ -1,147 +1,9 @@
+from __future__ import with_statement
+
 import copy
-import contextlib
 
-import parsing
-import evaluate
-import debug
-import builtin
-import settings
-
-
-class RecursionDecorator(object):
-    """
-    A decorator to detect recursions in statements. In a recursion a statement
-    at the same place, in the same module may not be executed two times.
-    """
-    def __init__(self, func):
-        self.func = func
-        self.reset()
-
-    def __call__(self, stmt, *args, **kwargs):
-        #print stmt, len(self.node_statements())
-        if self.push_stmt(stmt):
-            return []
-        else:
-            result = self.func(stmt, *args, **kwargs)
-            self.pop_stmt()
-        return result
-
-    def push_stmt(self, stmt):
-        self.current = RecursionNode(stmt, self.current)
-        if self._check_recursion():
-            debug.warning('catched recursion', stmt)
-            self.pop_stmt()
-            return True
-        return False
-
-    def pop_stmt(self):
-        if self.current is not None:
-            # I don't know how current can be None, but sometimes it happens
-            # with Python3.
-            self.current = self.current.parent
-
-    def _check_recursion(self):
-        test = self.current
-        while True:
-            test = test.parent
-            if self.current == test:
-                return True
-            if not test:
-                return False
-
-    def reset(self):
-        self.top = None
-        self.current = None
-
-    def node_statements(self):
-        result = []
-        n = self.current
-        while n:
-            result.insert(0, n.stmt)
-            n = n.parent
-        return result
-
-
-class RecursionNode(object):
-    """ A node of the RecursionDecorator. """
-    def __init__(self, stmt, parent):
-        self.script = stmt.get_parent_until()
-        self.position = stmt.start_pos
-        self.parent = parent
-        self.stmt = stmt
-
-        # Don't check param instances, they are not causing recursions
-        # The same's true for the builtins, because the builtins are really
-        # simple.
-        self.is_ignored = isinstance(stmt, parsing.Param) \
-                                   or (self.script == builtin.Builtin.scope)
-
-    def __eq__(self, other):
-        if not other:
-            return None
-        return self.script == other.script \
-                    and self.position == other.position \
-                    and not self.is_ignored and not other.is_ignored
-
-
-class ExecutionRecursionDecorator(object):
-    """
-    Catches recursions of executions.
-    It is designed like a Singelton. Only one instance should exist.
-    """
-    def __init__(self, func):
-        self.func = func
-        self.reset()
-
-    def __call__(self, execution, evaluate_generator=False):
-        debug.dbg('Execution recursions: %s' % execution, self.recursion_level,
-                            self.execution_count, len(self.execution_funcs))
-        if self.check_recursion(execution, evaluate_generator):
-            result = []
-        else:
-            result = self.func(execution, evaluate_generator)
-        self.cleanup()
-        return result
-
-    @classmethod
-    def cleanup(cls):
-        cls.parent_execution_funcs.pop()
-        cls.recursion_level -= 1
-
-    @classmethod
-    def check_recursion(cls, execution, evaluate_generator):
-        in_par_execution_funcs = execution.base in cls.parent_execution_funcs
-        in_execution_funcs = execution.base in cls.execution_funcs
-        cls.recursion_level += 1
-        cls.execution_count += 1
-        cls.execution_funcs.add(execution.base)
-        cls.parent_execution_funcs.append(execution.base)
-
-        if cls.execution_count > settings.max_executions:
-            return True
-
-        if isinstance(execution.base, (evaluate.Generator, evaluate.Array)):
-            return False
-        module = execution.get_parent_until()
-        if evaluate_generator or module == builtin.Builtin.scope:
-            return False
-
-        if in_par_execution_funcs:
-            if cls.recursion_level > settings.max_function_recursion_level:
-                return True
-        if in_execution_funcs and \
-                len(cls.execution_funcs) > settings.max_until_execution_unique:
-            return True
-        if cls.execution_count > settings.max_executions_without_builtins:
-            return True
-        return False
-
-    @classmethod
-    def reset(cls):
-        cls.recursion_level = 0
-        cls.parent_execution_funcs = []
-        cls.execution_funcs = set()
-        cls.execution_count = 0
+from jedi import common
+from jedi import parsing_representation as pr
 
 
 def fast_parent_copy(obj):
@@ -154,30 +16,40 @@ def fast_parent_copy(obj):
         new_obj = copy.copy(obj)
         new_elements[obj] = new_obj
 
-        items = new_obj.__dict__.items()
+        try:
+            items = list(new_obj.__dict__.items())
+        except AttributeError:
+            # __dict__ not available, because of __slots__
+            items = []
+
+        before = ()
+        for cls in new_obj.__class__.__mro__:
+            with common.ignored(AttributeError):
+                if before == cls.__slots__:
+                    continue
+                before = cls.__slots__
+                items += [(n, getattr(new_obj, n)) for n in before]
+
         for key, value in items:
             # replace parent (first try _parent and then parent)
-            if key in ['parent', '_parent', '_parent_stmt'] \
-                                                    and value is not None:
+            if key in ['parent', '_parent'] and value is not None:
                 if key == 'parent' and '_parent' in items:
                     # parent can be a property
                     continue
-                try:
+                with common.ignored(KeyError):
                     setattr(new_obj, key, new_elements[value])
-                except KeyError:
-                    pass
-            elif key in ['parent_stmt', 'parent_function']:
+            elif key in ['parent_function', 'use_as_parent', '_sub_module']:
                 continue
             elif isinstance(value, list):
                 setattr(new_obj, key, list_rec(value))
-            elif isinstance(value, (parsing.Simple, parsing.Call)):
+            elif isinstance(value, (pr.Simple, pr.Call)):
                 setattr(new_obj, key, recursion(value))
         return new_obj
 
     def list_rec(list_obj):
         copied_list = list_obj[:]   # lists, tuples, strings, unicode
         for i, el in enumerate(copied_list):
-            if isinstance(el, (parsing.Simple, parsing.Call)):
+            if isinstance(el, (pr.Simple, pr.Call)):
                 copied_list[i] = recursion(el)
             elif isinstance(el, list):
                 copied_list[i] = list_rec(el)
@@ -185,82 +57,81 @@ def fast_parent_copy(obj):
     return recursion(obj)
 
 
-def generate_param_array(args_tuple, parent_stmt=None):
-    """ This generates an array, that can be used as a param. """
-    values = []
-    for arg in args_tuple:
-        if arg is None:
-            values.append([])
+def check_arr_index(arr, pos):
+    positions = arr.arr_el_pos
+    for index, comma_pos in enumerate(positions):
+        if pos < comma_pos:
+            return index
+    return len(positions)
+
+
+def array_for_pos(stmt, pos, array_types=None):
+    """Searches for the array and position of a tuple"""
+    def search_array(arr, pos):
+        if arr.type == 'dict':
+            for stmt in arr.values + arr.keys:
+                new_arr, index = array_for_pos(stmt, pos, array_types)
+                if new_arr is not None:
+                    return new_arr, index
         else:
-            values.append([arg])
-    pos = None
-    arr = parsing.Array(pos, parsing.Array.TUPLE, parent_stmt, values=values)
-    evaluate.faked_scopes.append(arr)
-    return arr
+            for i, stmt in enumerate(arr):
+                new_arr, index = array_for_pos(stmt, pos, array_types)
+                if new_arr is not None:
+                    return new_arr, index
+                if arr.start_pos < pos <= stmt.end_pos:
+                    if not array_types or arr.type in array_types:
+                        return arr, i
+        if len(arr) == 0 and arr.start_pos < pos < arr.end_pos:
+            if not array_types or arr.type in array_types:
+                return arr, 0
+        return None, 0
+
+    def search_call(call, pos):
+        arr, index = None, 0
+        if call.next is not None:
+            if isinstance(call.next, pr.Array):
+                arr, index = search_array(call.next, pos)
+            else:
+                arr, index = search_call(call.next, pos)
+        if not arr and call.execution is not None:
+            arr, index = search_array(call.execution, pos)
+        return arr, index
+
+    if stmt.start_pos >= pos >= stmt.end_pos:
+        return None, 0
+
+    for command in stmt.get_commands():
+        arr = None
+        if isinstance(command, pr.Array):
+            arr, index = search_array(command, pos)
+        elif isinstance(command, pr.Call):
+            arr, index = search_call(command, pos)
+        if arr is not None:
+            return arr, index
+    return None, 0
 
 
-def scan_array_for_pos(arr, pos):
+def search_function_definition(stmt, pos):
     """
-    Returns the function Call that match search_name in an Array.
-    Makes changes to arr!
+    Returns the function Call that matches the position before.
     """
-    def check_arr_index():
-        positions = arr.arr_el_pos
-        for index, comma_pos in enumerate(positions):
-            if pos < comma_pos:
-                return index
-        return len(positions)
-
-    call = None
-    stop = False
-    for sub in arr.values:
-        call = None
-        for s in sub:
-            if isinstance(s, parsing.Array):
-                new = scan_array_for_pos(s, pos)
-                if new[0] is not None:
-                    call, index, stop = new
-                    if stop:
-                        return call, index, stop
-            elif isinstance(s, parsing.Call):
-                start_s = s
-                # check parts of calls
-                while s is not None:
-                    if s.start_pos >= pos:
-                        return call, check_arr_index(), stop
-                    elif s.execution is not None:
-                        end = s.execution.end_pos
-                        if s.execution.start_pos < pos and \
-                                (end is None or pos < end):
-                            c, index, stop = scan_array_for_pos(
-                                                    s.execution, pos)
-                            if stop:
-                                return c, index, stop
-
-                            # call should return without execution and
-                            # next
-                            reset = c or s
-                            if reset.execution.type not in \
-                                        [parsing.Array.TUPLE,
-                                        parsing.Array.NOARRAY]:
-                                return start_s, index, False
-
-                            reset.execution = None
-                            reset.next = None
-                            return c or start_s, index, True
-                    s = s.next
-
-    # The third return is just necessary for recursion inside, because
-    # it needs to know when to stop iterating.
-    return call, check_arr_index(), stop
+    # some parts will of the statement will be removed
+    stmt = fast_parent_copy(stmt)
+    arr, index = array_for_pos(stmt, pos, [pr.Array.TUPLE, pr.Array.NOARRAY])
+    if arr is not None and isinstance(arr.parent, pr.Call):
+        call = arr.parent
+        while isinstance(call.parent, pr.Call):
+            call = call.parent
+        arr.parent.execution = None
+        return call if call.type == pr.Call.NAME else None, index, False
+    return None, 0, False
 
 
-@contextlib.contextmanager
-def scale_speed_settings(factor):
-    a = settings.max_executions
-    b = settings.max_until_execution_unique
-    settings.max_executions *= factor
-    settings.max_until_execution_unique *= factor
-    yield
-    settings.max_executions = a
-    settings.max_until_execution_unique = b
+class FakeStatement(pr.Statement):
+    class SubModule():
+        line_offset = 0
+
+    def __init__(self, content):
+        cls = type(self)
+        p = 0, 0
+        super(cls, self).__init__(cls.SubModule, [], [], [content], p, p)

jedi/imports.py

@@ -1,55 +1,68 @@
+"""
+:mod:`imports` is here to resolve import statements and return the
+modules/classes/functions/whatever, which they stand for. However there's not
+any actual importing done. This module is about finding modules in the
+filesystem. This can be quite tricky sometimes, because Python imports are not
+always that simple.
+
+This module uses imp for python up to 3.2 and importlib for python 3.3 on; the
+correct implementation is delegated to _compatibility.
+
+This module also supports import autocompletion, which means to complete
+statements like ``from datetim`` (curser at the end would return ``datetime``).
+"""
 from __future__ import with_statement
 
 import os
 import pkgutil
-import imp
 import sys
-import time
-
-import builtin
-import modules
-import debug
-import parsing
-import evaluate
 import itertools
-import settings
+
+from jedi._compatibility import find_module
+from jedi import modules
+from jedi import common
+from jedi import debug
+from jedi import parsing_representation as pr
+from jedi import cache
+import builtin
+import evaluate
 
 # for debugging purposes only
 imports_processed = 0
 
-star_import_cache = {}
-
 
 class ModuleNotFound(Exception):
     pass
 
 
-class ImportPath(parsing.Base):
+class ImportPath(pr.Base):
     """
-    An ImportPath is the path of a `parsing.Import` object.
+    An ImportPath is the path of a `pr.Import` object.
     """
     class _GlobalNamespace(object):
+        def __init__(self):
+            self.start_pos = 0, 0
+            self.line_offset = 0
+
         def get_defined_names(self):
             return []
 
         def get_imports(self):
             return []
 
-        @property
-        def start_pos(self):
-            return (0, 0)
-
         def get_parent_until(self):
             return None
 
     GlobalNamespace = _GlobalNamespace()
 
     def __init__(self, import_stmt, is_like_search=False, kill_count=0,
-                                                    direct_resolve=False):
+                 direct_resolve=False, is_just_from=False):
         self.import_stmt = import_stmt
         self.is_like_search = is_like_search
         self.direct_resolve = direct_resolve
-        self.is_partial_import = bool(kill_count)
+        self.is_just_from = is_just_from
+
+        self.is_partial_import = bool(max(0, kill_count))
         path = import_stmt.get_parent_until().path
         self.file_path = os.path.dirname(path) if path is not None else None
 
@@ -58,7 +71,7 @@ class ImportPath(parsing.Base):
         if import_stmt.from_ns:
             self.import_path += import_stmt.from_ns.names
         if import_stmt.namespace:
-            if self.is_nested_import() and not direct_resolve:
+            if self._is_nested_import() and not direct_resolve:
                 self.import_path.append(import_stmt.namespace.names[0])
             else:
                 self.import_path += import_stmt.namespace.names
@@ -69,29 +82,30 @@ class ImportPath(parsing.Base):
     def __repr__(self):
         return '<%s: %s>' % (type(self).__name__, self.import_stmt)
 
-    def is_nested_import(self):
+    def _is_nested_import(self):
         """
         This checks for the special case of nested imports, without aliases and
-        from statement:
-        >>> import foo.bar
+        from statement::
+
+            import foo.bar
         """
         return not self.import_stmt.alias and not self.import_stmt.from_ns \
-                and len(self.import_stmt.namespace.names) > 1 \
-                and not self.direct_resolve
+            and len(self.import_stmt.namespace.names) > 1 \
+            and not self.direct_resolve
 
-    def get_nested_import(self, parent):
+    def _get_nested_import(self, parent):
         """
-        See documentation of `self.is_nested_import`.
+        See documentation of `self._is_nested_import`.
         Generates an Import statement, that can be used to fake nested imports.
         """
         i = self.import_stmt
         # This is not an existing Import statement. Therefore, set position to
         # 0 (0 is not a valid line number).
         zero = (0, 0)
-        n = parsing.Name(i.namespace.names[1:], zero, zero, self.import_stmt)
-        new = parsing.Import(zero, zero, n)
+        names = i.namespace.names[1:]
+        n = pr.Name(i._sub_module, names, zero, zero, self.import_stmt)
+        new = pr.Import(i._sub_module, zero, zero, n)
         new.parent = parent
-        evaluate.faked_scopes.append(new)
         debug.dbg('Generated a nested import: %s' % new)
         return new
 
@@ -99,24 +113,41 @@ class ImportPath(parsing.Base):
         names = []
         for scope in self.follow():
             if scope is ImportPath.GlobalNamespace:
-                if self.import_stmt.relative_count == 0:
-                    names += self.get_module_names()
+                if self._is_relative_import() == 0:
+                    names += self._get_module_names()
 
                 if self.file_path is not None:
                     path = os.path.abspath(self.file_path)
                     for i in range(self.import_stmt.relative_count - 1):
                         path = os.path.dirname(path)
-                    names += self.get_module_names([path])
+                    names += self._get_module_names([path])
+
+                    if self._is_relative_import():
+                        rel_path = self._get_relative_path() + '/__init__.py'
+                        with common.ignored(IOError):
+                            m = modules.Module(rel_path)
+                            names += m.parser.module.get_defined_names()
             else:
-                if on_import_stmt and isinstance(scope, parsing.Module) \
-                                        and scope.path.endswith('__init__.py'):
+                if on_import_stmt and isinstance(scope, pr.Module) \
+                        and scope.path.endswith('__init__.py'):
                     pkg_path = os.path.dirname(scope.path)
-                    names += self.get_module_names([pkg_path])
-                for s, scope_names in evaluate.get_names_for_scope(scope,
-                                                    include_builtin=False):
+                    paths = self._namespace_packages(pkg_path, self.import_path)
+                    names += self._get_module_names([pkg_path] + paths)
+                if self.is_just_from:
+                    # In the case of an import like `from x.` we don't need to
+                    # add all the variables.
+                    if ['os'] == self.import_path and not self._is_relative_import():
+                        # os.path is a hardcoded exception, because it's a
+                        # ``sys.modules`` modification.
+                        p = (0, 0)
+                        names.append(pr.Name(self.GlobalNamespace, [('path', p)],
+                                       p, p, self.import_stmt))
+                    continue
+                for s, scope_names in evaluate.get_names_of_scope(scope,
+                                                                  include_builtin=False):
                     for n in scope_names:
                         if self.import_stmt.from_ns is None \
-                                            or self.is_partial_import:
+                                or self.is_partial_import:
                                 # from_ns must be defined to access module
                                 # values plus a partial import means that there
                                 # is something after the import, which
@@ -126,23 +157,41 @@ class ImportPath(parsing.Base):
                         names.append(n)
         return names
 
-    def get_module_names(self, search_path=None):
+    def _get_module_names(self, search_path=None):
         """
         Get the names of all modules in the search_path. This means file names
         and not names defined in the files.
         """
-        if not search_path:
-            search_path = self.sys_path_with_modifications()
+        def generate_name(name):
+            return pr.Name(self.GlobalNamespace, [(name, inf_pos)],
+                           inf_pos, inf_pos, self.import_stmt)
+
         names = []
+        inf_pos = float('inf'), float('inf')
+        # add builtin module names
+        if search_path is None:
+            names += [generate_name(name) for name in sys.builtin_module_names]
+
+        if search_path is None:
+            search_path = self._sys_path_with_modifications()
         for module_loader, name, is_pkg in pkgutil.iter_modules(search_path):
-            inf_pos = (float('inf'), float('inf'))
-            names.append(parsing.Name([(name, inf_pos)], inf_pos, inf_pos,
-                                                            self.import_stmt))
+            names.append(generate_name(name))
         return names
 
-    def sys_path_with_modifications(self):
+    def _sys_path_with_modifications(self):
+        # If you edit e.g. gunicorn, there will be imports like this:
+        # `from gunicorn import something`. But gunicorn is not in the
+        # sys.path. Therefore look if gunicorn is a parent directory, #56.
+        in_path = []
+        if self.import_path:
+            parts = self.file_path.split(os.path.sep)
+            for i, p in enumerate(parts):
+                if p == self.import_path[0]:
+                    new = os.path.sep.join(parts[:i])
+                    in_path.append(new)
+
         module = self.import_stmt.get_parent_until()
-        return modules.sys_path_with_modifications(module)
+        return in_path + modules.sys_path_with_modifications(module)
 
     def follow(self, is_goto=False):
         """
@@ -161,25 +210,31 @@ class ImportPath(parsing.Base):
                 return []
 
             scopes = [scope]
-            scopes += itertools.chain.from_iterable(
-                            remove_star_imports(s) for s in scopes)
+            scopes += remove_star_imports(scope)
 
             # follow the rest of the import (not FS -> classes, functions)
             if len(rest) > 1 or rest and self.is_like_search:
                 scopes = []
+                if ['os', 'path'] == self.import_path[:2] \
+                        and not self._is_relative_import():
+                    # This is a huge exception, we follow a nested import
+                    # ``os.path``, because it's a very important one in Python
+                    # that is being achieved by messing with ``sys.modules`` in
+                    # ``os``.
+                    scopes = evaluate.follow_path(iter(rest), scope, scope)
             elif rest:
                 if is_goto:
                     scopes = itertools.chain.from_iterable(
-                        evaluate.get_scopes_for_name(s, rest[0], is_goto=True)
-                            for s in scopes)
+                        evaluate.find_name(s, rest[0], is_goto=True)
+                        for s in scopes)
                 else:
                     scopes = itertools.chain.from_iterable(
-                                        evaluate.follow_path(iter(rest), s, s)
-                                        for s in scopes)
+                        evaluate.follow_path(iter(rest), s, s)
+                        for s in scopes)
             scopes = list(scopes)
 
-            if self.is_nested_import():
-                scopes.append(self.get_nested_import(scope))
+            if self._is_nested_import():
+                scopes.append(self._get_nested_import(scope))
         else:
             scopes = [ImportPath.GlobalNamespace]
         debug.dbg('after import', scopes)
@@ -187,60 +242,122 @@ class ImportPath(parsing.Base):
         evaluate.follow_statement.pop_stmt()
         return scopes
 
+    def _is_relative_import(self):
+        return bool(self.import_stmt.relative_count)
+
+    def _get_relative_path(self):
+        path = self.file_path
+        for i in range(self.import_stmt.relative_count - 1):
+            path = os.path.dirname(path)
+        return path
+
+    def _namespace_packages(self, found_path, import_path):
+        """
+        Returns a list of paths of possible ``pkgutil``/``pkg_resources``
+        namespaces. If the package is no "namespace package", an empty list is
+        returned.
+        """
+        def follow_path(directories, paths):
+            try:
+                directory = next(directories)
+            except StopIteration:
+                return paths
+            else:
+                deeper_paths = []
+                for p in paths:
+                    new = os.path.join(p, directory)
+                    if os.path.isdir(new) and new != found_path:
+                        deeper_paths.append(new)
+                return follow_path(directories, deeper_paths)
+
+        with open(os.path.join(found_path, '__init__.py')) as f:
+            content = f.read()
+            # these are strings that need to be used for namespace packages,
+            # the first one is ``pkgutil``, the second ``pkg_resources``.
+            options = 'declare_namespace(__name__)', 'extend_path(__path__'
+            if options[0] in content or options[1] in content:
+                # It is a namespace, now try to find the rest of the modules.
+                return follow_path(iter(import_path), sys.path)
+        return []
+
     def _follow_file_system(self):
         """
         Find a module with a path (of the module, like usb.backend.libusb10).
         """
-        def follow_str(ns, string):
-            debug.dbg('follow_module', ns, string)
+        def follow_str(ns_path, string):
+            debug.dbg('follow_module', ns_path, string)
             path = None
-            if ns:
-                path = ns[1]
-            elif self.import_stmt.relative_count:
-                module = self.import_stmt.get_parent_until()
-                path = os.path.abspath(module.path)
-                for i in range(self.import_stmt.relative_count):
-                    path = os.path.dirname(path)
+            if ns_path:
+                path = ns_path
+            elif self._is_relative_import():
+                path = self._get_relative_path()
 
             global imports_processed
             imports_processed += 1
             if path is not None:
-                return imp.find_module(string, [path])
+                importing = find_module(string, [path])
             else:
                 debug.dbg('search_module', string, self.file_path)
                 # Override the sys.path. It works only good that way.
                 # Injecting the path directly into `find_module` did not work.
                 sys.path, temp = sys_path_mod, sys.path
                 try:
-                    i = imp.find_module(string)
-                except ImportError:
+                    importing = find_module(string)
+                finally:
                     sys.path = temp
-                    raise
-                sys.path = temp
-                return i
+
+            return importing
 
         if self.file_path:
-            sys_path_mod = list(self.sys_path_with_modifications())
-            sys_path_mod.insert(0, self.file_path)
+            sys_path_mod = list(self._sys_path_with_modifications())
+            module = self.import_stmt.get_parent_until()
+            if not module.has_explicit_absolute_import:
+                # If the module explicitly asks for absolute imports,
+                # there's probably a bogus local one.
+                sys_path_mod.insert(0, self.file_path)
         else:
-            sys_path_mod = list(builtin.get_sys_path())
+            sys_path_mod = list(modules.get_sys_path())
 
-        current_namespace = None
+        def module_not_found():
+            raise ModuleNotFound('The module you searched has not been found')
+
+        current_namespace = (None, None, None)
         # now execute those paths
         rest = []
         for i, s in enumerate(self.import_path):
             try:
-                current_namespace = follow_str(current_namespace, s)
+                current_namespace = follow_str(current_namespace[1], s)
             except ImportError:
-                if current_namespace:
-                    rest = self.import_path[i:]
-                else:
-                    raise ModuleNotFound(
-                            'The module you searched has not been found')
+                _continue = False
+                if self._is_relative_import() and len(self.import_path) == 1:
+                    # follow `from . import some_variable`
+                    rel_path = self._get_relative_path()
+                    with common.ignored(ImportError):
+                        current_namespace = follow_str(rel_path, '__init__')
+                elif current_namespace[2]:  # is a package
+                    for n in self._namespace_packages(current_namespace[1],
+                                                      self.import_path[:i]):
+                        try:
+                            current_namespace = follow_str(n, s)
+                            if current_namespace[1]:
+                                _continue = True
+                                break
+                        except ImportError:
+                            pass
+
+                if not _continue:
+                    if current_namespace[1]:
+                        rest = self.import_path[i:]
+                        break
+                    else:
+                        module_not_found()
+
+        if current_namespace == (None, None, False):
+            module_not_found()
 
         sys_path_mod.pop(0)  # TODO why is this here?
         path = current_namespace[1]
-        is_package_directory = current_namespace[2][2] == imp.PKG_DIRECTORY
+        is_package_directory = current_namespace[2]
 
         f = None
         if is_package_directory or current_namespace[0]:
@@ -255,9 +372,9 @@ class ImportPath(parsing.Base):
             if path.endswith('.py'):
                 f = modules.Module(path, source)
             else:
-                f = builtin.Parser(path=path)
+                f = builtin.BuiltinModule(path=path)
         else:
-            f = builtin.Parser(name=path)
+            f = builtin.BuiltinModule(name=path)
 
         return f.parser.module, rest
 
@@ -269,52 +386,15 @@ def strip_imports(scopes):
     """
     result = []
     for s in scopes:
-        if isinstance(s, parsing.Import):
+        if isinstance(s, pr.Import):
             result += ImportPath(s).follow()
         else:
             result.append(s)
     return result
 
 
-def cache_star_import(func):
-    def wrapper(scope, *args, **kwargs):
-        try:
-            mods = star_import_cache[scope]
-            if mods[0] + settings.star_import_cache_validity > time.time():
-                return mods[1]
-        except KeyError:
-            pass
-        # cache is too old and therefore invalid or not available
-        invalidate_star_import_cache(scope)
-        mods = func(scope, *args, **kwargs)
-        star_import_cache[scope] = time.time(), mods
-
-        return mods
-    return wrapper
-
-
-def invalidate_star_import_cache(module, only_main=False):
-    """ Important if some new modules are being reparsed """
-    try:
-        t, mods = star_import_cache[module]
-
-        del star_import_cache[module]
-
-        for m in mods:
-            invalidate_star_import_cache(m, only_main=True)
-    except KeyError:
-        pass
-
-    if not only_main:
-        # We need a list here because otherwise the list is being changed
-        # during the iteration in py3k: iteritems -> items.
-        for key, (t, mods) in list(star_import_cache.items()):
-            if module in mods:
-                invalidate_star_import_cache(key)
-
-
-@cache_star_import
-def remove_star_imports(scope, ignored_modules=[]):
+@cache.cache_star_import
+def remove_star_imports(scope, ignored_modules=()):
     """
     Check a module for star imports:
     >>> from module import *

jedi/interpret.py

@@ -0,0 +1,171 @@
+"""
+Module to handle interpreted Python objects.
+"""
+
+import itertools
+import tokenize
+
+from jedi import parsing_representation as pr
+
+
+class ObjectImporter(object):
+
+    """
+    Import objects in "raw" namespace such as :func:`locals`.
+    """
+
+    def __init__(self, scope):
+        self.scope = scope
+
+        count = itertools.count()
+        self._genname = lambda: '*jedi-%s*' % next(count)
+        """
+        Generate unique variable names to avoid name collision.
+        To avoid name collision to already defined names, generated
+        names are invalid as Python identifier.
+        """
+
+    def import_raw_namespace(self, raw_namespace):
+        """
+        Import interpreted Python objects in a namespace.
+
+        Three kinds of objects are treated here.
+
+        1. Functions and classes.  The objects imported like this::
+
+               from os.path import join
+
+        2. Modules.  The objects imported like this::
+
+               import os
+
+        3. Instances.  The objects created like this::
+
+               from datetime import datetime
+               dt = datetime(2013, 1, 1)
+
+        :type raw_namespace: dict
+        :arg  raw_namespace: e.g., the dict given by `locals`
+        """
+        scope = self.scope
+        for (variable, obj) in raw_namespace.items():
+            objname = getattr(obj, '__name__', None)
+
+            # Import functions and classes
+            module = getattr(obj, '__module__', None)
+            if module and objname:
+                fakeimport = self.make_fakeimport(module, objname, variable)
+                scope.add_import(fakeimport)
+                continue
+
+            # Import modules
+            if getattr(obj, '__file__', None) and objname:
+                fakeimport = self.make_fakeimport(objname)
+                scope.add_import(fakeimport)
+                continue
+
+            # Import instances
+            objclass = getattr(obj, '__class__', None)
+            module = getattr(objclass, '__module__', None)
+            if objclass and module:
+                alias = self._genname()
+                fakeimport = self.make_fakeimport(module, objclass.__name__,
+                                                  alias)
+                fakestmt = self.make_fakestatement(variable, alias, call=True)
+                scope.add_import(fakeimport)
+                scope.add_statement(fakestmt)
+                continue
+
+    def make_fakeimport(self, module, variable=None, alias=None):
+        """
+        Make a fake import object.
+
+        The following statements are created depending on what parameters
+        are given:
+
+        - only `module`: ``import <module>``
+        - `module` and `variable`: ``from <module> import <variable>``
+        - all: ``from <module> import <variable> as <alias>``
+
+        :type   module: str
+        :arg    module: ``<module>`` part in ``from <module> import ...``
+        :type variable: str
+        :arg  variable: ``<variable>`` part in ``from ... import <variable>``
+        :type    alias: str
+        :arg     alias: ``<alias>`` part in ``... import ... as <alias>``.
+
+        :rtype: :class:`parsing_representation.Import`
+        """
+        submodule = self.scope._sub_module
+        if variable:
+            varname = pr.Name(
+                module=submodule,
+                names=[(variable, (-1, 0))],
+                start_pos=(-1, 0),
+                end_pos=(None, None))
+        else:
+            varname = None
+        modname = pr.Name(
+            module=submodule,
+            names=[(module, (-1, 0))],
+            start_pos=(-1, 0),
+            end_pos=(None, None))
+        if alias:
+            aliasname = pr.Name(
+                module=submodule,
+                names=[(alias, (-1, 0))],
+                start_pos=(-1, 0),
+                end_pos=(None, None))
+        else:
+            aliasname = None
+        if varname:
+            fakeimport = pr.Import(
+                module=submodule,
+                namespace=varname,
+                from_ns=modname,
+                alias=aliasname,
+                start_pos=(-1, 0),
+                end_pos=(None, None))
+        else:
+            fakeimport = pr.Import(
+                module=submodule,
+                namespace=modname,
+                alias=aliasname,
+                start_pos=(-1, 0),
+                end_pos=(None, None))
+        return fakeimport
+
+    def make_fakestatement(self, lhs, rhs, call=False):
+        """
+        Make a fake statement object that represents ``lhs = rhs``.
+
+        :type call: bool
+        :arg  call: When `call` is true, make a fake statement that represents
+                   ``lhs = rhs()``.
+
+        :rtype: :class:`parsing_representation.Statement`
+        """
+        submodule = self.scope._sub_module
+        lhsname = pr.Name(
+            module=submodule,
+            names=[(lhs, (0, 0))],
+            start_pos=(0, 0),
+            end_pos=(None, None))
+        rhsname = pr.Name(
+            module=submodule,
+            names=[(rhs, (0, 0))],
+            start_pos=(0, 0),
+            end_pos=(None, None))
+        token_list = [lhsname, (tokenize.OP, '=', (0, 0)), rhsname]
+        if call:
+            token_list.extend([
+                (tokenize.OP, '(', (0, 0)),
+                (tokenize.OP, ')', (0, 0)),
+            ])
+        return pr.Statement(
+            module=submodule,
+            set_vars=[lhsname],
+            used_vars=[rhsname],
+            token_list=token_list,
+            start_pos=(0, 0),
+            end_pos=(None, None))

jedi/keywords.py

@@ -1,18 +1,18 @@
-import keyword
-
-from _compatibility import is_py3k
-import builtin
+from __future__ import with_statement
 
 import pydoc
+import keyword
+
+from jedi._compatibility import is_py3k
+from jedi import parsing_representation as pr
+from jedi import common
+import builtin
+
 try:
     from pydoc_data import topics as pydoc_topics
 except ImportError:
     # Python 2.6
-    try:
-        import pydoc_topics
-    except ImportError:
-        # Python 2.5
-        pydoc_topics = None
+    import pydoc_topics
 
 if is_py3k:
     keys = keyword.kwlist
@@ -20,7 +20,7 @@ else:
     keys = keyword.kwlist + ['None', 'False', 'True']
 
 
-def get_keywords(string='', pos=(0, 0), all=False):
+def keywords(string='', pos=(0, 0), all=False):
     if all:
         return set([Keyword(k, pos) for k in keys])
     if string in keys:
@@ -28,6 +28,15 @@ def get_keywords(string='', pos=(0, 0), all=False):
     return set()
 
 
+def keyword_names(*args, **kwargs):
+    kwds = []
+    for k in keywords(*args, **kwargs):
+        start = k.start_pos
+        end = start[0], start[1] + len(k.name)
+        kwds.append(pr.Name(k.parent, [(k.name, start)], start, end, k))
+    return kwds
+
+
 def get_operator(string, pos):
     return Keyword(string, pos)
 
@@ -63,12 +72,10 @@ def imitate_pydoc(string):
     # with unicode strings)
     string = str(string)
     h = pydoc.help
-    try:
+    with common.ignored(KeyError):
         # try to access symbols
         string = h.symbols[string]
         string, _, related = string.partition(' ')
-    except KeyError:
-        pass
 
     get_target = lambda s: h.topics.get(s, h.keywords.get(s))
     while isinstance(string, str):

jedi/mixin/_sqlite3.pym

@@ -0,0 +1,26 @@
+def connect(database, timeout=None, isolation_level=None, detect_types=None, factory=None):
+    return Connection()
+
+
+class Connection():
+    def cursor(self):
+        return Cursor()
+
+
+class Cursor():
+    def cursor(self):
+        return Cursor()
+
+    def fetchone(self):
+        return Row()
+
+    def fetchmany(self, size=cursor.arraysize):
+        return [self.fetchone()]
+
+    def fetchall(self):
+        return [self.fetchone()]
+
+
+class Row():
+    def keys(self):
+        return ['']

jedi/mixin/_sre.pym

@@ -22,7 +22,7 @@ def compile():
         def expand(self):
             return ''
 
-        def group(self):
+        def group(self, nr):
             return ''
 
         def groupdict(self):
@@ -37,14 +37,14 @@ def compile():
         groups = 0
         pattern = 'a'
 
-        def findall(self):
+        def findall(self, string, pos=None, endpos=None):
             """
             findall(string[, pos[, endpos]]) --> list.
             Return a list of all non-overlapping matches of pattern in string.
             """
             return ['a']
 
-        def finditer(self):
+        def finditer(self, string, pos=None, endpos=None):
             """
             finditer(string[, pos[, endpos]]) --> iterator.
             Return an iterator over all non-overlapping matches for the
@@ -53,7 +53,7 @@ def compile():
             """
             yield SRE_Match(self)
 
-        def match(self):
+        def match(self, string, pos=None, endpos=None):
             """
             match(string[, pos[, endpos]]) --> match object or None.
             Matches zero or more characters at the beginning of the string
@@ -61,10 +61,10 @@ def compile():
             """
             return SRE_Match(self)
 
-        def scanner(self):
+        def scanner(self, string, pos=None, endpos=None):
             pass
 
-        def search(self):
+        def search(self, string, pos=None, endpos=None):
             """
             search(string[, pos[, endpos]]) --> match object or None.
             Scan through string looking for a match, and return a corresponding
@@ -72,14 +72,14 @@ def compile():
             """
             return SRE_Match(self)
 
-        def split(self):
+        def split(self, string, maxsplit=0]):
             """
             split(string[, maxsplit = 0])  --> list.
             Split string by the occurrences of pattern.
             """
             return ['a']
 
-        def sub(self):
+        def sub(self, repl, string, count=0):
             """
             sub(repl, string[, count = 0]) --> newstring
             Return the string obtained by replacing the leftmost non-overlapping
@@ -87,7 +87,7 @@ def compile():
             """
             return ''
 
-        def subn(self):
+        def subn(self, repl, string, count=0):
             """
             subn(repl, string[, count = 0]) --> (newstring, number of subs)
             Return the tuple (new_string, number_of_subs_made) found by replacing

jedi/mixin/builtins.pym

@@ -142,6 +142,18 @@ class set():
     def copy(self):
         return self
 
+    def difference(self, other):
+        return self - other
+
+    def intersection(self, other):
+        return self & other
+
+    def symmetric_difference(self, other):
+        return self ^ other
+
+    def union(self, other):
+        return self | other
+
 
 class frozenset():
     def __init__(self, iterable=[]):

jedi/modules.py

@@ -1,31 +1,77 @@
+"""
+Don't confuse these classes with :mod:`parsing_representation` modules, the
+modules here can access these representation with ``module.parser.module``.
+``Module`` exists mainly for caching purposes.
+
+Basically :mod:`modules` offers the classes:
+
+- ``CachedModule``, a base class for Cachedmodule.
+- ``Module`` the class for all normal Python modules (not builtins, they are at
+  home at :mod:`builtin`).
+- ``ModuleWithCursor``, holds the module information for :class:`api.Script`.
+
+Apart from those classes there's a ``sys.path`` fetching function, as well as
+`Virtual Env` and `Django` detection.
+"""
 from __future__ import with_statement
 
-from _compatibility import exec_function
-
 import re
-import tokenize
+import tokenizer as tokenize
 import sys
 import os
-import time
+from ast import literal_eval
 
-import parsing
-import builtin
-import debug
-import evaluate
-import settings
-import imports
+from jedi._compatibility import exec_function, unicode
+from jedi import cache
+from jedi import parsing_representation as pr
+from jedi import fast_parser
+from jedi import debug
+from jedi import common
 
 
-class Module(builtin.CachedModule):
+class CachedModule(object):
+    """
+    The base type for all modules, which is not to be confused with
+    `parsing_representation.Module`. Caching happens here.
+    """
+
+    def __init__(self, path=None, name=None):
+        self.path = path and os.path.abspath(path)
+        self.name = name
+        self._parser = None
+
+    @property
+    def parser(self):
+        """ get the parser lazy """
+        if self._parser is None:
+            self._parser = cache.load_module(self.path, self.name) \
+                or self._load_module()
+        return self._parser
+
+    def _get_source(self):
+        raise NotImplementedError()
+
+    def _load_module(self):
+        source = self._get_source()
+        p = self.path or self.name
+        p = fast_parser.FastParser(source, p)
+        cache.save_module(self.path, self.name, p)
+        return p
+
+
+class Module(CachedModule):
     """
     Manages all files, that are parsed and caches them.
 
     :param path: The module path of the file.
     :param source: The source code of the file.
     """
-    def __init__(self, path, source):
+    def __init__(self, path, source=None):
         super(Module, self).__init__(path=path)
-        self.source = source
+        if source is None:
+            with open(path) as f:
+                source = f.read()
+        self.source = source_to_unicode(source)
         self._line_cache = None
 
     def _get_source(self):
@@ -49,67 +95,62 @@ class ModuleWithCursor(Module):
     def __init__(self, path, source, position):
         super(ModuleWithCursor, self).__init__(path, source)
         self.position = position
+        self.source = source
+        self._path_until_cursor = None
 
         # this two are only used, because there is no nonlocal in Python 2
         self._line_temp = None
         self._relevant_temp = None
 
-        self.source = source
-        self._part_parser = None
-
     @property
     def parser(self):
         """ get the parser lazy """
         if not self._parser:
-            try:
-                ts, parser = builtin.CachedModule.cache[self.path]
-                imports.invalidate_star_import_cache(parser.module)
-
-                del builtin.CachedModule.cache[self.path]
-            except KeyError:
-                pass
+            with common.ignored(KeyError):
+                parser = cache.parser_cache[self.path].parser
+                cache.invalidate_star_import_cache(parser.module)
             # Call the parser already here, because it will be used anyways.
             # Also, the position is here important (which will not be used by
             # default), therefore fill the cache here.
-            self._parser = parsing.PyFuzzyParser(self.source, self.path,
-                                                                self.position)
-            if self.path is not None:
-                builtin.CachedModule.cache[self.path] = time.time(), \
-                                                        self._parser
+            self._parser = fast_parser.FastParser(self.source, self.path,
+                                                  self.position)
+            # don't pickle that module, because it's changing fast
+            cache.save_module(self.path, self.name, self._parser,
+                              pickling=False)
         return self._parser
 
     def get_path_until_cursor(self):
         """ Get the path under the cursor. """
-        result = self._get_path_until_cursor()
-        self._start_cursor_pos = self._line_temp + 1, self._column_temp
-        return result
+        if self._path_until_cursor is None:  # small caching
+            self._path_until_cursor, self._start_cursor_pos = \
+                self._get_path_until_cursor(self.position)
+        return self._path_until_cursor
 
     def _get_path_until_cursor(self, start_pos=None):
         def fetch_line():
-            line = self.get_line(self._line_temp)
             if self._is_first:
                 self._is_first = False
                 self._line_length = self._column_temp
-                line = line[:self._column_temp]
+                line = self._first_line
             else:
+                line = self.get_line(self._line_temp)
                 self._line_length = len(line)
                 line = line + '\n'
             # add lines with a backslash at the end
-            while 1:
+            while True:
                 self._line_temp -= 1
                 last_line = self.get_line(self._line_temp)
+                #print self._line_temp, repr(last_line)
                 if last_line and last_line[-1] == '\\':
                     line = last_line[:-1] + ' ' + line
+                    self._line_length = len(last_line)
                 else:
                     break
             return line[::-1]
 
         self._is_first = True
-        if start_pos is None:
-            self._line_temp = self.position[0]
-            self._column_temp = self.position[1]
-        else:
-            self._line_temp, self._column_temp = start_pos
+        self._line_temp, self._column_temp = start_cursor = start_pos
+        self._first_line = self.get_line(self._line_temp)[:self._column_temp]
 
         open_brackets = ['(', '[', '{']
         close_brackets = [')', ']', '}']
@@ -118,9 +159,13 @@ class ModuleWithCursor(Module):
         string = ''
         level = 0
         force_point = False
+        last_type = None
         try:
             for token_type, tok, start, end, line in gen:
-                #print 'tok', token_type, tok, force_point
+                # print 'tok', token_type, tok, force_point
+                if last_type == token_type == tokenize.NAME:
+                    string += ' '
+
                 if level > 0:
                     if tok in close_brackets:
                         level += 1
@@ -142,51 +187,79 @@ class ModuleWithCursor(Module):
                 elif token_type == tokenize.NUMBER:
                     pass
                 else:
+                    self._column_temp = self._line_length - end[1]
                     break
 
+                x = start_pos[0] - end[0] + 1
+                l = self.get_line(x)
+                l = self._first_line if x == start_pos[0] else l
+                start_cursor = x, len(l) - end[1]
                 self._column_temp = self._line_length - end[1]
                 string += tok
+                last_type = token_type
         except tokenize.TokenError:
             debug.warning("Tokenize couldn't finish", sys.exc_info)
 
-        return string[::-1]
+        # string can still contain spaces at the end
+        return string[::-1].strip(), start_cursor
 
     def get_path_under_cursor(self):
         """
         Return the path under the cursor. If there is a rest of the path left,
         it will be added to the stuff before it.
         """
+        return self.get_path_until_cursor() + self.get_path_after_cursor()
+
+    def get_path_after_cursor(self):
         line = self.get_line(self.position[0])
-        after = re.search("[\w\d]*", line[self.position[1]:]).group(0)
-        return self.get_path_until_cursor() + after
+        return re.search("[\w\d]*", line[self.position[1]:]).group(0)
 
     def get_operator_under_cursor(self):
         line = self.get_line(self.position[0])
         after = re.match("[^\w\s]+", line[self.position[1]:])
         before = re.match("[^\w\s]+", line[:self.position[1]][::-1])
         return (before.group(0) if before is not None else '') \
-                + (after.group(0) if after is not None else '')
+            + (after.group(0) if after is not None else '')
 
-    def get_context(self):
+    def get_context(self, yield_positions=False):
         pos = self._start_cursor_pos
-        while pos > (1, 0):
+        while True:
             # remove non important white space
             line = self.get_line(pos[0])
-            while pos[1] > 0 and line[pos[1] - 1].isspace():
-                pos = pos[0], pos[1] - 1
+            while True:
+                if pos[1] == 0:
+                    line = self.get_line(pos[0] - 1)
+                    if line and line[-1] == '\\':
+                        pos = pos[0] - 1, len(line) - 1
+                        continue
+                    else:
+                        break
+
+                if line[pos[1] - 1].isspace():
+                    pos = pos[0], pos[1] - 1
+                else:
+                    break
 
             try:
-                yield self._get_path_until_cursor(start_pos=pos)
+                result, pos = self._get_path_until_cursor(start_pos=pos)
+                if yield_positions:
+                    yield pos
+                else:
+                    yield result
             except StopIteration:
-                yield ''
-            pos = self._line_temp, self._column_temp
-
-        while True:
-            yield ''
+                if yield_positions:
+                    yield None
+                else:
+                    yield ''
 
     def get_line(self, line_nr):
         if not self._line_cache:
-            self._line_cache = self.source.split('\n')
+            self._line_cache = self.source.splitlines()
+            if self.source:
+                if self.source[-1] == '\n':
+                    self._line_cache.append('')
+            else:  # ''.splitlines() == []
+                self._line_cache = ['']
 
         if line_nr == 0:
             # This is a fix for the zeroth line. We need a newline there, for
@@ -199,23 +272,26 @@ class ModuleWithCursor(Module):
         except IndexError:
             raise StopIteration()
 
-    def get_part_parser(self):
-        """ Returns a parser that contains only part of the source code. This
-        exists only because of performance reasons.
-        """
-        if self._part_parser:
-            return self._part_parser
-
-        # TODO check for docstrings
-        length = settings.part_line_length
-        offset = max(self.position[0] - length, 0)
-        s = '\n'.join(self.source.split('\n')[offset:offset + length])
-        self._part_parser = parsing.PyFuzzyParser(s, self.path, self.position,
-                                                        line_offset=offset)
-        return self._part_parser
+    def get_position_line(self):
+        return self.get_line(self.position[0])[:self.position[1]]
 
 
-@evaluate.memoize_default([])
+def get_sys_path():
+    def check_virtual_env(sys_path):
+        """ Add virtualenv's site-packages to the `sys.path`."""
+        venv = os.getenv('VIRTUAL_ENV')
+        if not venv:
+            return
+        venv = os.path.abspath(venv)
+        p = os.path.join(
+            venv, 'lib', 'python%d.%d' % sys.version_info[:2], 'site-packages')
+        sys_path.insert(0, p)
+
+    check_virtual_env(sys.path)
+    return [p for p in sys.path if p != ""]
+
+
+@cache.memoize_default([])
 def sys_path_with_modifications(module):
     def execute_code(code):
         c = "import os; from os.path import *; result=%s"
@@ -238,16 +314,18 @@ def sys_path_with_modifications(module):
         try:
             possible_stmts = module.used_names['path']
         except KeyError:
-            return builtin.get_sys_path()
+            return get_sys_path()
 
-        sys_path = list(builtin.get_sys_path())  # copy
+        sys_path = list(get_sys_path())  # copy
         for p in possible_stmts:
-            try:
-                call = p.get_assignment_calls().get_only_subelement()
-            except AttributeError:
+            if not isinstance(p, pr.Statement):
                 continue
+            commands = p.get_commands()
+            if len(commands) != 1:  # sys.path command is just one thing.
+                continue
+            call = commands[0]
             n = call.name
-            if not isinstance(n, parsing.Name) or len(n.names) != 3:
+            if not isinstance(n, pr.Name) or len(n.names) != 3:
                 continue
             if n.names[:2] != ('sys', 'path'):
                 continue
@@ -260,7 +338,7 @@ def sys_path_with_modifications(module):
                 continue
 
             if array_cmd == 'insert':
-                exe_type, exe.type = exe.type, parsing.Array.NOARRAY
+                exe_type, exe.type = exe.type, pr.Array.NOARRAY
                 exe_pop = exe.values.pop(0)
                 res = execute_code(exe.get_code())
                 if res is not None:
@@ -276,13 +354,13 @@ def sys_path_with_modifications(module):
         return sys_path
 
     if module.path is None:
-        return []  # support for modules without a path is intentionally bad.
+        # Support for modules without a path is bad, therefore return the
+        # normal path.
+        return list(get_sys_path())
 
     curdir = os.path.abspath(os.curdir)
-    try:
+    with common.ignored(OSError):
         os.chdir(os.path.dirname(module.path))
-    except OSError:
-        pass
 
     result = check_module(module)
     result += detect_django_path(module.path)
@@ -303,10 +381,37 @@ def detect_django_path(module_path):
         else:
             module_path = new
 
-        try:
+        with common.ignored(IOError):
             with open(module_path + os.path.sep + 'manage.py'):
                 debug.dbg('Found django path: %s' % module_path)
                 result.append(module_path)
-        except IOError:
-            pass
     return result
+
+
+def source_to_unicode(source, encoding=None):
+    def detect_encoding():
+        """ For the implementation of encoding definitions in Python, look at:
+        http://www.python.org/dev/peps/pep-0263/
+        http://docs.python.org/2/reference/lexical_analysis.html#encoding-\
+                                                                declarations
+        """
+        byte_mark = literal_eval(r"b'\xef\xbb\xbf'")
+        if source.startswith(byte_mark):
+            # UTF-8 byte-order mark
+            return 'utf-8'
+
+        first_two_lines = re.match(r'(?:[^\n]*\n){0,2}', str(source)).group(0)
+        possible_encoding = re.search(r"coding[=:]\s*([-\w.]+)",
+                                      first_two_lines)
+        if possible_encoding:
+            return possible_encoding.group(1)
+        else:
+            # the default if nothing else has been set -> PEP 263
+            return encoding if encoding is not None else 'iso-8859-1'
+
+    if isinstance(source, unicode):
+        # only cast str/bytes
+        return source
+
+    # cast to unicode by default
+    return unicode(source, detect_encoding(), 'replace')

jedi/recursion.py

@@ -0,0 +1,156 @@
+"""
+Recursions are the recipe of |jedi| to conquer Python code. However, someone
+must stop recursions going mad. Some settings are here to make |jedi| stop at
+the right time. You can read more about them :ref:`here <settings-recursion>`.
+
+Next to :mod:`cache` this module also makes |jedi| not thread-safe. Why?
+``ExecutionRecursionDecorator`` uses class variables to count the function
+calls.
+"""
+from jedi import parsing_representation as pr
+from jedi import debug
+from jedi import settings
+import evaluate_representation as er
+import builtin
+
+
+class RecursionDecorator(object):
+    """
+    A decorator to detect recursions in statements. In a recursion a statement
+    at the same place, in the same module may not be executed two times.
+    """
+    def __init__(self, func):
+        self.func = func
+        self.reset()
+
+    def __call__(self, stmt, *args, **kwargs):
+        # print stmt, len(self.node_statements())
+        if self.push_stmt(stmt):
+            return []
+        else:
+            result = self.func(stmt, *args, **kwargs)
+            self.pop_stmt()
+        return result
+
+    def push_stmt(self, stmt):
+        self.current = RecursionNode(stmt, self.current)
+        check = self._check_recursion()
+        if check:  # TODO remove False!!!!
+            debug.warning('catched stmt recursion: %s against %s @%s'
+                          % (stmt, check.stmt, stmt.start_pos))
+            self.pop_stmt()
+            return True
+        return False
+
+    def pop_stmt(self):
+        if self.current is not None:
+            # I don't know how current can be None, but sometimes it happens
+            # with Python3.
+            self.current = self.current.parent
+
+    def _check_recursion(self):
+        test = self.current
+        while True:
+            test = test.parent
+            if self.current == test:
+                return test
+            if not test:
+                return False
+
+    def reset(self):
+        self.top = None
+        self.current = None
+
+    def node_statements(self):
+        result = []
+        n = self.current
+        while n:
+            result.insert(0, n.stmt)
+            n = n.parent
+        return result
+
+
+class RecursionNode(object):
+    """ A node of the RecursionDecorator. """
+    def __init__(self, stmt, parent):
+        self.script = stmt.get_parent_until()
+        self.position = stmt.start_pos
+        self.parent = parent
+        self.stmt = stmt
+
+        # Don't check param instances, they are not causing recursions
+        # The same's true for the builtins, because the builtins are really
+        # simple.
+        self.is_ignored = isinstance(stmt, pr.Param) \
+            or (self.script == builtin.Builtin.scope)
+
+    def __eq__(self, other):
+        if not other:
+            return None
+
+        is_list_comp = lambda x: isinstance(x, pr.ForFlow) and x.is_list_comp
+        return self.script == other.script \
+            and self.position == other.position \
+            and not is_list_comp(self.stmt.parent) \
+            and not is_list_comp(other.parent) \
+            and not self.is_ignored and not other.is_ignored
+
+
+class ExecutionRecursionDecorator(object):
+    """
+    Catches recursions of executions.
+    It is designed like a Singelton. Only one instance should exist.
+    """
+    def __init__(self, func):
+        self.func = func
+        self.reset()
+
+    def __call__(self, execution, evaluate_generator=False):
+        debug.dbg('Execution recursions: %s' % execution, self.recursion_level,
+                  self.execution_count, len(self.execution_funcs))
+        if self.check_recursion(execution, evaluate_generator):
+            result = []
+        else:
+            result = self.func(execution, evaluate_generator)
+        self.cleanup()
+        return result
+
+    @classmethod
+    def cleanup(cls):
+        cls.parent_execution_funcs.pop()
+        cls.recursion_level -= 1
+
+    @classmethod
+    def check_recursion(cls, execution, evaluate_generator):
+        in_par_execution_funcs = execution.base in cls.parent_execution_funcs
+        in_execution_funcs = execution.base in cls.execution_funcs
+        cls.recursion_level += 1
+        cls.execution_count += 1
+        cls.execution_funcs.add(execution.base)
+        cls.parent_execution_funcs.append(execution.base)
+
+        if cls.execution_count > settings.max_executions:
+            return True
+
+        if isinstance(execution.base, (er.Generator, er.Array)):
+            return False
+        module = execution.get_parent_until()
+        if evaluate_generator or module == builtin.Builtin.scope:
+            return False
+
+        if in_par_execution_funcs:
+            if cls.recursion_level > settings.max_function_recursion_level:
+                return True
+        if in_execution_funcs and \
+                len(cls.execution_funcs) > settings.max_until_execution_unique:
+            return True
+        if cls.execution_count > settings.max_executions_without_builtins:
+            return True
+        return False
+
+    @classmethod
+    def reset(cls):
+        cls.recursion_level = 0
+        cls.parent_execution_funcs = []
+        cls.execution_funcs = set()
+        cls.execution_count = 0

jedi/refactoring.py

@@ -0,0 +1,205 @@
+"""
+Introduce some basic refactoring functions to |jedi|. This module is still in a
+very early development stage and needs much testing and improvement.
+
+.. warning:: I won't do too much here, but if anyone wants to step in, please
+             do. Refactoring is none of my priorities
+
+It uses the |jedi| `API <plugin-api.html>`_ and supports currently the
+following functions (sometimes bug-prone):
+
+- rename
+- extract variable
+- inline variable
+"""
+from __future__ import with_statement
+
+import difflib
+
+from jedi import common
+from jedi import modules
+from jedi import helpers
+from jedi import parsing_representation as pr
+
+
+class Refactoring(object):
+    def __init__(self, change_dct):
+        """
+        :param change_dct: dict(old_path=(new_path, old_lines, new_lines))
+        """
+        self.change_dct = change_dct
+
+    def old_files(self):
+        dct = {}
+        for old_path, (new_path, old_l, new_l) in self.change_dct.items():
+            dct[new_path] = '\n'.join(new_l)
+        return dct
+
+    def new_files(self):
+        dct = {}
+        for old_path, (new_path, old_l, new_l) in self.change_dct.items():
+            dct[new_path] = '\n'.join(new_l)
+        return dct
+
+    def diff(self):
+        texts = []
+        for old_path, (new_path, old_l, new_l) in self.change_dct.items():
+            if old_path:
+                udiff = difflib.unified_diff(old_l, new_l)
+            else:
+                udiff = difflib.unified_diff(old_l, new_l, old_path, new_path)
+            texts.append('\n'.join(udiff))
+        return '\n'.join(texts)
+
+
+def rename(script, new_name):
+    """ The `args` / `kwargs` params are the same as in `api.Script`.
+    :param operation: The refactoring operation to execute.
+    :type operation: str
+    :type source: str
+    :return: list of changed lines/changed files
+    """
+    return Refactoring(_rename(script.usages(), new_name))
+
+
+def _rename(names, replace_str):
+    """ For both rename and inline. """
+    order = sorted(names, key=lambda x: (x.module_path, x.line, x.column),
+                   reverse=True)
+
+    def process(path, old_lines, new_lines):
+        if new_lines is not None:  # goto next file, save last
+            dct[path] = path, old_lines, new_lines
+
+    dct = {}
+    current_path = object()
+    new_lines = old_lines = None
+    for name in order:
+        if name.in_builtin_module():
+            continue
+        if current_path != name.module_path:
+            current_path = name.module_path
+
+            process(current_path, old_lines, new_lines)
+            if current_path is not None:
+                # None means take the source that is a normal param.
+                with open(current_path) as f:
+                    source = f.read()
+
+            new_lines = modules.source_to_unicode(source).splitlines()
+            old_lines = new_lines[:]
+
+        nr, indent = name.line, name.column
+        line = new_lines[nr - 1]
+        new_lines[nr - 1] = line[:indent] + replace_str + \
+            line[indent + len(name.text):]
+    process(current_path, old_lines, new_lines)
+    return dct
+
+
+def extract(script, new_name):
+    """ The `args` / `kwargs` params are the same as in `api.Script`.
+    :param operation: The refactoring operation to execute.
+    :type operation: str
+    :type source: str
+    :return: list of changed lines/changed files
+    """
+    new_lines = modules.source_to_unicode(script.source).splitlines()
+    old_lines = new_lines[:]
+
+    user_stmt = script._parser.user_stmt
+
+    # TODO care for multiline extracts
+    dct = {}
+    if user_stmt:
+        pos = script._pos
+        line_index = pos[0] - 1
+        arr, index = helpers.array_for_pos(user_stmt, pos)
+        if arr is not None:
+            start_pos = arr[index].start_pos
+            end_pos = arr[index].end_pos
+
+            # take full line if the start line is different from end line
+            e = end_pos[1] if end_pos[0] == start_pos[0] else None
+            start_line = new_lines[start_pos[0] - 1]
+            text = start_line[start_pos[1]:e]
+            for l in range(start_pos[0], end_pos[0] - 1):
+                text += '\n' + l
+            if e is None:
+                end_line = new_lines[end_pos[0] - 1]
+                text += '\n' + end_line[:end_pos[1]]
+
+            # remove code from new lines
+            t = text.lstrip()
+            del_start = start_pos[1] + len(text) - len(t)
+
+            text = t.rstrip()
+            del_end = len(t) - len(text)
+            if e is None:
+                new_lines[end_pos[0] - 1] = end_line[end_pos[1] - del_end:]
+                e = len(start_line)
+            else:
+                e = e - del_end
+            start_line = start_line[:del_start] + new_name + start_line[e:]
+            new_lines[start_pos[0] - 1] = start_line
+            new_lines[start_pos[0]:end_pos[0] - 1] = []
+
+            # add parentheses in multiline case
+            open_brackets = ['(', '[', '{']
+            close_brackets = [')', ']', '}']
+            if '\n' in text and not (text[0] in open_brackets and text[-1] ==
+                                     close_brackets[open_brackets.index(text[0])]):
+                text = '(%s)' % text
+
+            # add new line before statement
+            indent = user_stmt.start_pos[1]
+            new = "%s%s = %s" % (' ' * indent, new_name, text)
+            new_lines.insert(line_index, new)
+    dct[script.path] = script.path, old_lines, new_lines
+    return Refactoring(dct)
+
+
+def inline(script):
+    """
+    :type script: api.Script
+    """
+    new_lines = modules.source_to_unicode(script.source).splitlines()
+
+    dct = {}
+
+    definitions = script.goto_assignments()
+    with common.ignored(AssertionError):
+        assert len(definitions) == 1
+        stmt = definitions[0]._definition
+        usages = script.usages()
+        inlines = [r for r in usages
+                   if not stmt.start_pos <= (r.line, r.column) <= stmt.end_pos]
+        inlines = sorted(inlines, key=lambda x: (x.module_path, x.line, x.column),
+                         reverse=True)
+        commands = stmt.get_commands()
+        # don't allow multiline refactorings for now.
+        assert stmt.start_pos[0] == stmt.end_pos[0]
+        index = stmt.start_pos[0] - 1
+
+        line = new_lines[index]
+        replace_str = line[commands[0].start_pos[1]:stmt.end_pos[1] + 1]
+        replace_str = replace_str.strip()
+        # tuples need parentheses
+        if commands and isinstance(commands[0], pr.Array):
+            arr = commands[0]
+            if replace_str[0] not in ['(', '[', '{'] and len(arr) > 1:
+                replace_str = '(%s)' % replace_str
+
+        # if it's the only assignment, remove the statement
+        if len(stmt.set_vars) == 1:
+            line = line[:stmt.start_pos[1]] + line[stmt.end_pos[1]:]
+
+        dct = _rename(inlines, replace_str)
+        # remove the empty line
+        new_lines = dct[script.path][2]
+        if line.strip():
+            new_lines[index] = line
+        else:
+            new_lines.pop(index)
+
+    return Refactoring(dct)

jedi/replstartup.py

@@ -0,0 +1,23 @@
+"""
+To use Jedi completion in Python interpreter, add the following in your shell
+setup (e.g., ``.bashrc``)::
+
+    export PYTHONSTARTUP="$(python -m jedi repl)"
+
+Then you will be able to use Jedi completer in your Python interpreter::
+
+    $ python
+    Python 2.7.2+ (default, Jul 20 2012, 22:15:08)
+    [GCC 4.6.1] on linux2
+    Type "help", "copyright", "credits" or "license" for more information.
+    >>> import os
+    >>> os.path.join().split().in<TAB>                     # doctest: +SKIP
+    os.path.join().split().index   os.path.join().split().insert
+
+"""
+
+import jedi.utils
+jedi.utils.setup_readline()
+del jedi
+# Note: try not to do many things here, as it will contaminate global
+# namespace of the interpreter.

jedi/settings.py

@@ -1,87 +1,236 @@
+"""
+This module contains variables with global |jedi| setting. To change the
+behavior of |jedi|, change the variables defined in
+:mod:`jedi.settings`.
+
+Plugins should expose an interface so that the user can adjust the
+configuration.
+
+
+Example usage::
+
+    from jedi import settings
+    settings.case_insensitive_completion = True
+
+
+Completion output
+~~~~~~~~~~~~~~~~~
+
+.. autodata:: case_insensitive_completion
+.. autodata:: add_dot_after_module
+.. autodata:: add_bracket_after_function
+.. autodata:: no_completion_duplicates
+
+
+Filesystem cache
+~~~~~~~~~~~~~~~~
+
+.. autodata:: cache_directory
+.. autodata:: use_filesystem_cache
+
+
+Parser
+~~~~~~
+
+.. autodata:: fast_parser
+
+
+Dynamic stuff
+~~~~~~~~~~~~~
+
+.. autodata:: dynamic_arrays_instances
+.. autodata:: dynamic_array_additions
+.. autodata:: dynamic_params
+.. autodata:: dynamic_params_for_other_modules
+.. autodata:: additional_dynamic_modules
+
+
+.. _settings-recursion:
+
+Recursions
+~~~~~~~~~~
+
+Recursion settings are important if you don't want extremly
+recursive python code to go absolutely crazy. First of there is a
+global limit :data:`max_executions`. This limit is important, to set
+a maximum amount of time, the completion may use.
+
+The default values are based on experiments while completing the |jedi| library
+itself (inception!). But I don't think there's any other Python library that
+uses recursion in a similarly extreme way. These settings make the completion
+definitely worse in some cases. But a completion should also be fast.
+
+.. autodata:: max_until_execution_unique
+.. autodata:: max_function_recursion_level
+.. autodata:: max_executions_without_builtins
+.. autodata:: max_executions
+.. autodata:: scale_function_definition
+
+
+Caching
+~~~~~~~
+
+.. autodata:: star_import_cache_validity
+.. autodata:: function_definition_validity
+
+
+"""
+import os
+import platform
+
 # ----------------
 # completion output settings
 # ----------------
 
-# The completion is by default case insensitive.
 case_insensitive_completion = True
+"""
+The completion is by default case insensitive.
+"""
 
-# Adds a dot after a module, because a module that is not accessed this way is
-# definitely not the normal case. However, in VIM this doesn't work, that's why
-# it isn't used at the moment.
 add_dot_after_module = False
+"""
+Adds a dot after a module, because a module that is not accessed this way is
+definitely not the normal case. However, in VIM this doesn't work, that's why
+it isn't used at the moment.
+"""
 
-# Adds an opening bracket after a function, because that's normal behaviour.
-# Removed it again, because in VIM that is not very practical.
 add_bracket_after_function = False
+"""
+Adds an opening bracket after a function, because that's normal behaviour.
+Removed it again, because in VIM that is not very practical.
+"""
 
+no_completion_duplicates = True
+"""
+If set, completions with the same name don't appear in the output anymore,
+but are in the `same_name_completions` attribute.
+"""
+
+# ----------------
+# Filesystem cache
+# ----------------
+
+use_filesystem_cache = True
+"""
+Use filesystem cache to save once parsed files with pickle.
+"""
+
+if platform.system().lower() == 'windows':
+    _cache_directory = os.path.join(os.getenv('APPDATA') or '~', 'Jedi',
+                                    'Jedi')
+elif platform.system().lower() == 'darwin':
+    _cache_directory = os.path.join('~', 'Library', 'Caches', 'Jedi')
+else:
+    _cache_directory = os.path.join(os.getenv('XDG_CACHE_HOME') or '~/.cache',
+                                    'jedi')
+cache_directory = os.path.expanduser(_cache_directory)
+"""
+The path where all the caches can be found.
+
+On Linux, this defaults to ``~/.cache/jedi/``, on OS X to
+``~/Library/Caches/Jedi/`` and on Windows to ``%APPDATA%\\Jedi\\Jedi\\``.
+On Linux, if environment variable ``$XDG_CACHE_HOME`` is set,
+``$XDG_CACHE_HOME/jedi`` is used instead of the default one.
+"""
+
+# ----------------
+# parser
+# ----------------
+
+fast_parser = True
+"""
+Use the fast parser. This means that reparsing is only being done if
+something has been changed e.g. to a function. If this happens, only the
+function is being reparsed.
+"""
 
 # ----------------
 # dynamic stuff
 # ----------------
 
-# check for `append`, etc. on array instances like list()
 dynamic_arrays_instances = True
-# check for `append`, etc. on arrays: [], {}, ()
+"""
+Check for `append`, etc. on array instances like list()
+"""
+
 dynamic_array_additions = True
+"""
+check for `append`, etc. on arrays: [], {}, ()
+"""
 
-# A dynamic param completion, finds the callees of the function, which define
-# the params of a function.
 dynamic_params = True
-# Do the same for other modules.
-dynamic_params_for_other_modules = True
+"""
+A dynamic param completion, finds the callees of the function, which define
+the params of a function.
+"""
+
+dynamic_params_for_other_modules = True
+"""
+Do the same for other modules.
+"""
 
-# Additional modules in which Jedi checks if statements are to be found. This
-# is practical for IDE's, that want to administrate their modules themselves.
 additional_dynamic_modules = []
+"""
+Additional modules in which |jedi| checks if statements are to be found. This
+is practical for IDEs, that want to administrate their modules themselves.
+"""
+
+dynamic_flow_information = True
+"""
+Check for `isinstance` and other information to infer a type.
+"""
 
 # ----------------
 # recursions
 # ----------------
 
-# Recursion settings are important if you don't want extremly recursive python
-# code to go absolutely crazy. First of there is a global limit
-# `max_executions`. This limit is important, to set a maximum amount of time,
-# the completion may use.
-#
-# The `max_until_execution_unique` limit is probably the most important one,
-# because if that limit is passed, functions can only be one time executed. So
-# new functions will be executed, complex recursions with the same functions
-# again and again, are ignored.
-#
-# `max_function_recursion_level` is more about whether the recursions are
-# stopped in deepth or in width. The ratio beetween this and
-# `max_until_execution_unique` is important here. It stops a recursion (after
-# the number of function calls in the recursion), if it was already used
-# earlier.
-#
-# The values are based on my experimental tries, used on the jedi library. But
-# I don't think there's any other Python library, that uses recursion in a
-# similar (extreme) way. This makes the completion definitely worse in some
-# cases. But a completion should also be fast.
+max_until_execution_unique = 50
+"""
+This limit is probably the most important one, because if this limit is
+exceeded, functions can only be one time executed. So new functions will be
+executed, complex recursions with the same functions again and again, are
+ignored.
+"""
 
 max_function_recursion_level = 5
-max_until_execution_unique = 50
+"""
+`max_function_recursion_level` is more about whether the recursions are
+stopped in deepth or in width. The ratio beetween this and
+`max_until_execution_unique` is important here. It stops a recursion (after
+the number of function calls in the recursion), if it was already used
+earlier.
+"""
+
 max_executions_without_builtins = 200
+"""
+.. todo:: Document this.
+"""
+
 max_executions = 250
+"""
+A maximum amount of time, the completion may use.
+"""
 
-# Because get_in_function_call is normally used on every single key hit, it has
-# to be faster than a normal completion. This is the factor that is used to
-# scale `max_executions` and `max_until_execution_unique`:
-scale_get_in_function_call = 0.1
+scale_function_definition = 0.1
+"""
+Because function_definition is normally used on every single key hit, it has
+to be faster than a normal completion. This is the factor that is used to
+scale `max_executions` and `max_until_execution_unique`:
+"""
 
 # ----------------
-# various
+# caching validity (time)
 # ----------------
 
-# Size of the current code part, which is used to speed up parsing.
-part_line_length = 20
-
-# ----------------
-# star import caching
-# ----------------
-
-# In huge packages like numpy, checking all star imports on every completion
-# might be slow, therefore we do a star import caching, that lasts a certain
-# time span (in seconds).
-
 star_import_cache_validity = 60.0
+"""
+In huge packages like numpy, checking all star imports on every completion
+might be slow, therefore we do a star import caching, that lasts a certain
+time span (in seconds).
+"""
+
+function_definition_validity = 3.0
+"""
+Finding function calls might be slow (0.1-0.5s). This is not acceptible for
+normal writing. Therefore cache it for a short time.
+"""

jedi/tokenizer.py

@@ -0,0 +1,286 @@
+"""
+This tokenizer has been copied from the ``tokenize.py`` standard library
+tokenizer. The reason was simple: The standanrd library  tokenizer fails
+if the indentation is not right. The fast parser of jedi however requires
+"wrong" indentation.
+
+Basically this is a stripped down version of the standard library module, so
+you can read the documentation there.
+"""
+
+import string
+import re
+from token import *
+from codecs import lookup, BOM_UTF8
+import collections
+cookie_re = re.compile("coding[:=]\s*([-\w.]+)")
+
+namechars = string.ascii_letters + '_'
+
+
+COMMENT = N_TOKENS
+tok_name[COMMENT] = 'COMMENT'
+NL = N_TOKENS + 1
+tok_name[NL] = 'NL'
+ENCODING = N_TOKENS + 2
+tok_name[ENCODING] = 'ENCODING'
+N_TOKENS += 3
+
+
+class TokenInfo(collections.namedtuple('TokenInfo', 'type string start end line')):
+    def __repr__(self):
+        annotated_type = '%d (%s)' % (self.type, tok_name[self.type])
+        return ('TokenInfo(type=%s, string=%r, start=%r, end=%r, line=%r)' %
+                self._replace(type=annotated_type))
+
+
+def group(*choices):
+    return '(' + '|'.join(choices) + ')'
+
+
+def any(*choices):
+    return group(*choices) + '*'
+
+
+def maybe(*choices):
+    return group(*choices) + '?'
+
+
+# Note: we use unicode matching for names ("\w") but ascii matching for
+# number literals.
+Whitespace = r'[ \f\t]*'
+Comment = r'#[^\r\n]*'
+Ignore = Whitespace + any(r'\\\r?\n' + Whitespace) + maybe(Comment)
+Name = r'\w+'
+
+Hexnumber = r'0[xX][0-9a-fA-F]+'
+Binnumber = r'0[bB][01]+'
+Octnumber = r'0[oO][0-7]+'
+Decnumber = r'(?:0+|[1-9][0-9]*)'
+Intnumber = group(Hexnumber, Binnumber, Octnumber, Decnumber)
+Exponent = r'[eE][-+]?[0-9]+'
+Pointfloat = group(r'[0-9]+\.[0-9]*', r'\.[0-9]+') + maybe(Exponent)
+Expfloat = r'[0-9]+' + Exponent
+Floatnumber = group(Pointfloat, Expfloat)
+Imagnumber = group(r'[0-9]+[jJ]', Floatnumber + r'[jJ]')
+Number = group(Imagnumber, Floatnumber, Intnumber)
+
+# Tail end of ' string.
+Single = r"[^'\\]*(?:\\.[^'\\]*)*'"
+# Tail end of " string.
+Double = r'[^"\\]*(?:\\.[^"\\]*)*"'
+# Tail end of ''' string.
+Single3 = r"[^'\\]*(?:(?:\\.|'(?!''))[^'\\]*)*'''"
+# Tail end of """ string.
+Double3 = r'[^"\\]*(?:(?:\\.|"(?!""))[^"\\]*)*"""'
+Triple = group("[bB]?[rR]?'''", '[bB]?[rR]?"""')
+# Single-line ' or " string.
+String = group(r"[bB]?[rR]?'[^\n'\\]*(?:\\.[^\n'\\]*)*'",
+               r'[bB]?[rR]?"[^\n"\\]*(?:\\.[^\n"\\]*)*"')
+
+# Because of leftmost-then-longest match semantics, be sure to put the
+# longest operators first (e.g., if = came before ==, == would get
+# recognized as two instances of =).
+Operator = group(r"\*\*=?", r">>=?", r"<<=?", r"!=",
+                 r"//=?", r"->",
+                 r"[+\-*/%&|^=<>]=?",
+                 r"~")
+
+Bracket = '[][(){}]'
+Special = group(r'\r?\n', r'\.\.\.', r'[:;.,@]')
+Funny = group(Operator, Bracket, Special)
+
+PlainToken = group(Number, Funny, String, Name)
+Token = Ignore + PlainToken
+
+# First (or only) line of ' or " string.
+ContStr = group(r"[bB]?[rR]?'[^\n'\\]*(?:\\.[^\n'\\]*)*" +
+                group("'", r'\\\r?\n'),
+                r'[bB]?[rR]?"[^\n"\\]*(?:\\.[^\n"\\]*)*' +
+                group('"', r'\\\r?\n'))
+PseudoExtras = group(r'\\\r?\n', Comment, Triple)
+PseudoToken = Whitespace + group(PseudoExtras, Number, Funny, ContStr, Name)
+
+
+def _compile(expr):
+    return re.compile(expr, re.UNICODE)
+
+
+tokenprog, pseudoprog, single3prog, double3prog = map(
+    _compile, (Token, PseudoToken, Single3, Double3))
+endprogs = {"'": _compile(Single), '"': _compile(Double),
+            "'''": single3prog, '"""': double3prog,
+            "r'''": single3prog, 'r"""': double3prog,
+            "b'''": single3prog, 'b"""': double3prog,
+            "br'''": single3prog, 'br"""': double3prog,
+            "R'''": single3prog, 'R"""': double3prog,
+            "B'''": single3prog, 'B"""': double3prog,
+            "bR'''": single3prog, 'bR"""': double3prog,
+            "Br'''": single3prog, 'Br"""': double3prog,
+            "BR'''": single3prog, 'BR"""': double3prog,
+            'r': None, 'R': None, 'b': None, 'B': None}
+
+triple_quoted = {}
+for t in ("'''", '"""',
+          "r'''", 'r"""', "R'''", 'R"""',
+          "b'''", 'b"""', "B'''", 'B"""',
+          "br'''", 'br"""', "Br'''", 'Br"""',
+          "bR'''", 'bR"""', "BR'''", 'BR"""'):
+    triple_quoted[t] = t
+single_quoted = {}
+for t in ("'", '"',
+          "r'", 'r"', "R'", 'R"',
+          "b'", 'b"', "B'", 'B"',
+          "br'", 'br"', "Br'", 'Br"',
+          "bR'", 'bR"', "BR'", 'BR"'):
+    single_quoted[t] = t
+
+del _compile
+
+tabsize = 8
+
+
+class TokenError(Exception):
+    pass
+
+
+def generate_tokens(readline):
+    lnum = parenlev = continued = 0
+    numchars = '0123456789'
+    contstr, needcont = '', 0
+    contline = None
+    indents = [0]
+
+    while True:             # loop over lines in stream
+        try:
+            line = readline()
+        except StopIteration:
+            line = b''
+
+        lnum += 1
+        pos, max = 0, len(line)
+
+        if contstr:                            # continued string
+            if not line:
+                # multiline string has not been finished
+                break
+            endmatch = endprog.match(line)
+            if endmatch:
+                pos = end = endmatch.end(0)
+                yield TokenInfo(STRING, contstr + line[:end],
+                                strstart, (lnum, end), contline + line)
+                contstr, needcont = '', 0
+                contline = None
+            elif needcont and line[-2:] != '\\\n' and line[-3:] != '\\\r\n':
+                yield TokenInfo(ERRORTOKEN, contstr + line,
+                                strstart, (lnum, len(line)), contline)
+                contstr = ''
+                contline = None
+                continue
+            else:
+                contstr = contstr + line
+                contline = contline + line
+                continue
+
+        elif parenlev == 0 and not continued:  # new statement
+            if not line:
+                break
+            column = 0
+            while pos < max:                   # measure leading whitespace
+                if line[pos] == ' ':
+                    column += 1
+                elif line[pos] == '\t':
+                    column = (column // tabsize + 1) * tabsize
+                elif line[pos] == '\f':
+                    column = 0
+                else:
+                    break
+                pos += 1
+            if pos == max:
+                break
+
+            if line[pos] in '#\r\n':           # skip comments or blank lines
+                if line[pos] == '#':
+                    comment_token = line[pos:].rstrip('\r\n')
+                    nl_pos = pos + len(comment_token)
+                    yield TokenInfo(COMMENT, comment_token,
+                                   (lnum, pos), (lnum, pos + len(comment_token)), line)
+                    yield TokenInfo(NL, line[nl_pos:],
+                                   (lnum, nl_pos), (lnum, len(line)), line)
+                else:
+                    yield TokenInfo(
+                        (NL, COMMENT)[line[pos] == '#'], line[pos:],
+                        (lnum, pos), (lnum, len(line)), line)
+                continue
+
+            if column > indents[-1]:           # count indents or dedents
+                indents.append(column)
+                yield TokenInfo(INDENT, line[:pos], (lnum, 0), (lnum, pos), line)
+            while column < indents[-1]:
+                indents = indents[:-1]
+                yield TokenInfo(DEDENT, '', (lnum, pos), (lnum, pos), line)
+
+        else:                                  # continued statement
+            if not line:
+                # basically a statement has not been finished here.
+                break
+            continued = 0
+
+        while pos < max:
+            pseudomatch = pseudoprog.match(line, pos)
+            if pseudomatch:                                # scan for tokens
+                start, end = pseudomatch.span(1)
+                spos, epos, pos = (lnum, start), (lnum, end), end
+                token, initial = line[start:end], line[start]
+
+                if (initial in numchars or                  # ordinary number
+                        (initial == '.' and token != '.' and token != '...')):
+                    yield TokenInfo(NUMBER, token, spos, epos, line)
+                elif initial in '\r\n':
+                    yield TokenInfo(NL if parenlev > 0 else NEWLINE,
+                                    token, spos, epos, line)
+                elif initial == '#':
+                    assert not token.endswith("\n")
+                    yield TokenInfo(COMMENT, token, spos, epos, line)
+                elif token in triple_quoted:
+                    endprog = endprogs[token]
+                    endmatch = endprog.match(line, pos)
+                    if endmatch:                           # all on one line
+                        pos = endmatch.end(0)
+                        token = line[start:pos]
+                        yield TokenInfo(STRING, token, spos, (lnum, pos), line)
+                    else:
+                        strstart = (lnum, start)           # multiple lines
+                        contstr = line[start:]
+                        contline = line
+                        break
+                elif initial in single_quoted or \
+                        token[:2] in single_quoted or \
+                        token[:3] in single_quoted:
+                    if token[-1] == '\n':                  # continued string
+                        strstart = (lnum, start)
+                        endprog = (endprogs[initial] or endprogs[token[1]] or
+                                   endprogs[token[2]])
+                        contstr, needcont = line[start:], 1
+                        contline = line
+                        break
+                    else:                                  # ordinary string
+                        yield TokenInfo(STRING, token, spos, epos, line)
+                elif initial in namechars:                 # ordinary name
+                    yield TokenInfo(NAME, token, spos, epos, line)
+                elif initial == '\\':                      # continued stmt
+                    continued = 1
+                else:
+                    if initial in '([{':
+                        parenlev += 1
+                    elif initial in ')]}':
+                        parenlev -= 1
+                    yield TokenInfo(OP, token, spos, epos, line)
+            else:
+                yield TokenInfo(ERRORTOKEN, line[pos],
+                               (lnum, pos), (lnum, pos + 1), line)
+                pos += 1
+
+    for indent in indents[1:]:                 # pop remaining indent levels
+        yield TokenInfo(DEDENT, '', (lnum, 0), (lnum, 0), '')
+    yield TokenInfo(ENDMARKER, '', (lnum, 0), (lnum, 0), '')

jedi/utils.py

@@ -0,0 +1,97 @@
+"""
+Utilities for end-users.
+"""
+
+from __future__ import absolute_import
+import __main__
+
+from jedi import Interpreter
+
+
+def setup_readline(namespace_module=__main__):
+    """
+    Install Jedi completer to :mod:`readline`.
+
+    This function setups :mod:`readline` to use Jedi in Python interactive
+    shell.  If you want to use a custom ``PYTHONSTARTUP`` file (typically
+    ``$HOME/.pythonrc.py``), you can add this piece of code::
+
+        try:
+            from jedi.utils import setup_readline
+            setup_readline()
+        except ImportError:
+            # Fallback to the stdlib readline completer if it is installed.
+            # Taken from http://docs.python.org/2/library/rlcompleter.html
+            print("Jedi is not installed, falling back to readline")
+            try:
+                import readline
+                import rlcompleter
+                readline.parse_and_bind("tab: complete")
+            except ImportError:
+                print("Readline is not installed either. No tab completion is enabled.")
+
+    This will fallback to the readline completer if Jedi is not installed.
+    The readline completer will only complete names in the global namespace,
+    so for example,
+
+    >>> ran<TAB> # doctest: +SKIP
+
+    will complete to ``range``
+
+    with both Jedi and readline, but
+
+    >>> range(10).cou<TAB> # doctest: +SKIP
+
+    will show complete to ``range(10).count`` only with Jedi.
+
+    You'll also need to add ``export PYTHONSTARTUP=$HOME/.pythonrc.py`` to
+    your shell profile (usually ``.bash_profile`` or ``.profile`` if you use
+    bash).
+
+    """
+    class JediRL():
+        def complete(self, text, state):
+            """
+            This complete stuff is pretty weird, a generator would make
+            a lot more sense, but probably due to backwards compatibility
+            this is still the way how it works.
+
+            The only important part is stuff in the ``state == 0`` flow,
+            everything else has been copied from the ``rlcompleter`` std.
+            library module.
+            """
+            if state == 0:
+                import os, sys
+                sys.path.insert(0, os.getcwd())
+                # Calling python doesn't have a path, so add to sys.path.
+                try:
+                    interpreter = Interpreter(text, [namespace_module.__dict__])
+
+                    path, dot, like = interpreter._get_completion_parts()
+                    before = text[:len(text) - len(like)]
+                    completions = interpreter.completions()
+                finally:
+                    sys.path.pop(0)
+
+                self.matches = [before + c.name_with_symbols for c in completions]
+            try:
+                return self.matches[state]
+            except IndexError:
+                return None
+
+    try:
+        import readline
+    except ImportError:
+        print("Module readline not available.")
+    else:
+        readline.set_completer(JediRL().complete)
+        readline.parse_and_bind("tab: complete")
+        # jedi itself does the case matching
+        readline.parse_and_bind("set completion-ignore-case on")
+        # because it's easier to hit the tab just once
+        readline.parse_and_bind("set show-all-if-unmodified")
+        readline.parse_and_bind("set show-all-if-ambiguous on")
+        # don't repeat all the things written in the readline all the time
+        readline.parse_and_bind("set completion-prefix-display-length 2")
+        # No delimiters, Jedi handles that.
+        readline.set_completer_delims('')

pytest.ini

@@ -0,0 +1,10 @@
+[pytest]
+addopts = --doctest-modules
+
+# Ignore broken files in blackbox test directories
+norecursedirs = .* docs completion refactor absolute_import namespace_package
+
+# Activate `clean_jedi_cache` fixture for all tests.  This should be
+# fine as long as we are using `clean_jedi_cache` as a session scoped
+# fixture.
+usefixtures = clean_jedi_cache

setup.py

@@ -1,33 +1,46 @@
 #!/usr/bin/env python
 
-from setuptools import setup
+from __future__ import with_statement
+try:
+    from setuptools import setup
+except ImportError:
+    # Distribute is not actually required to install
+    from distutils.core import setup
 
 __AUTHOR__ = 'David Halter'
 __AUTHOR_EMAIL__ = 'davidhalter88@gmail.com'
 
-readme = open('README.rst').read()
+readme = open('README.rst').read() + '\n\n' + open('CHANGELOG.rst').read()
 
+import jedi
+VERSION = '.'.join(str(x) for x in jedi.__version__)
 
 setup(name='jedi',
-      version='0.5b3',
+      version=VERSION,
       description='An autocompletion tool for Python that can be used for text editors.',
       author=__AUTHOR__,
       author_email=__AUTHOR_EMAIL__,
       maintainer=__AUTHOR__,
       maintainer_email=__AUTHOR_EMAIL__,
       url='https://github.com/davidhalter/jedi',
-      license='LGPLv3',
+      license='MIT',
       keywords='python completion refactoring vim',
       long_description=readme,
-      packages=['jedi', 'jedi.mixin'],
+      packages=['jedi'],
+      package_data={'jedi': ['mixin/*.pym']},
       platforms=['any'],
       classifiers=[
           'Development Status :: 4 - Beta',
           'Environment :: Plugins',
           'Intended Audience :: Developers',
-          'License :: OSI Approved :: GNU Library or Lesser General Public License (LGPL)',
+          'License :: OSI Approved :: MIT License',
           'Operating System :: OS Independent',
-          'Programming Language :: Python',
+          'Programming Language :: Python :: 2',
+          'Programming Language :: Python :: 2.6',
+          'Programming Language :: Python :: 2.7',
+          'Programming Language :: Python :: 3',
+          'Programming Language :: Python :: 3.2',
+          'Programming Language :: Python :: 3.3',
           'Topic :: Software Development :: Libraries :: Python Modules',
           'Topic :: Text Editors :: Integrated Development Environments (IDE)',
           'Topic :: Utilities',

sith.py

@@ -0,0 +1,221 @@
+#!/usr/bin/env python
+
+"""
+Sith attacks (and helps debugging) Jedi.
+
+Randomly search Python files and run Jedi on it.  Exception and used
+arguments are recorded to ``./record.json`` (specified by --record)::
+
+    ./sith.py random /path/to/sourcecode
+
+Redo recorded exception::
+
+    ./sith.py redo
+
+Show recorded exception::
+
+    ./sith.py show
+
+Run a specific operation
+
+    ./sith.py run <operation> </path/to/source/file.py> <line> <col>
+
+Where operation is one of completions, goto_assignments, goto_definitions,
+usages, or call_signatures.
+
+Note: Line numbers start at 1; columns start at 0 (this is consistent with
+many text editors, including Emacs).
+
+Usage:
+  sith.py [--pdb|--ipdb|--pudb] [-d] [-n=<nr>] [-f] [--record=<file>] random [<path>]
+  sith.py [--pdb|--ipdb|--pudb] [-d] [-f] [--record=<file>] redo
+  sith.py [--pdb|--ipdb|--pudb] [-d] [-f] run <operation> <path> <line> <column>
+  sith.py show [--record=<file>]
+  sith.py -h | --help
+
+Options:
+  -h --help             Show this screen.
+  --record=<file>       Exceptions are recorded in here [default: record.json].
+  -f, --fs-cache        By default, file system cache is off for reproducibility.
+  -n, --maxtries=<nr>   Maximum of random tries [default: 100]
+  -d, --debug           Jedi print debugging when an error is raised.
+  --pdb                 Launch pdb when error is raised.
+  --ipdb                Launch ipdb when error is raised.
+  --pudb                Launch pudb when error is raised.
+"""
+
+from __future__ import print_function, division, unicode_literals
+from docopt import docopt
+
+import json
+import os
+import random
+import sys
+import traceback
+
+import jedi
+
+
+class SourceFinder(object):
+    _files = None
+
+    @staticmethod
+    def fetch(file_path):
+        if not os.path.isdir(file_path):
+            yield file_path
+            return
+        for root, dirnames, filenames in os.walk(file_path):
+            for name in filenames:
+                if name.endswith('.py'):
+                    yield os.path.join(root, name)
+
+    @classmethod
+    def files(cls, file_path):
+        if cls._files is None:
+            cls._files = list(cls.fetch(file_path))
+        return cls._files
+
+
+class TestCase(object):
+    def __init__(self, operation, path, line, column, traceback=None):
+        if operation not in self.operations:
+            raise ValueError("%s is not a valid operation" % operation)
+
+        # Set other attributes
+        self.operation = operation
+        self.path = path
+        self.line = line
+        self.column = column
+        self.traceback = traceback
+
+    @classmethod
+    def from_cache(cls, record):
+        with open(record) as f:
+            args = json.load(f)
+        return cls(*args)
+
+    operations = [
+        'completions', 'goto_assignments', 'goto_definitions', 'usages',
+        'call_signatures']
+
+    @classmethod
+    def generate(cls, file_path):
+        operation = random.choice(cls.operations)
+
+        path = random.choice(SourceFinder.files(file_path))
+        with open(path) as f:
+            source = f.read()
+            lines = source.splitlines()
+
+        if not lines:
+            lines = ['']
+        line = random.randint(1, len(lines))
+        column = random.randint(0, len(lines[line - 1]))
+        return cls(operation, path, line, column)
+
+    def run(self, debugger, record=None, print_result=False):
+        try:
+            with open(self.path) as f:
+                self.script = jedi.Script(f.read(), self.line, self.column, self.path)
+            self.completions = getattr(self.script, self.operation)()
+            if print_result:
+                self.show_location(self.line, self.column)
+                self.show_operation()
+        except jedi.NotFoundError:
+            pass
+        except Exception:
+            self.traceback = traceback.format_exc()
+            if record is not None:
+                call_args = (self.operation, self.path, self.line, self.column, self.traceback)
+                with open(record, 'w') as f:
+                    json.dump(call_args, f)
+            self.show_errors()
+            if debugger:
+                einfo = sys.exc_info()
+                pdb = __import__(debugger)
+                if debugger == 'pudb':
+                    pdb.post_mortem(einfo[2], einfo[0], einfo[1])
+                else:
+                    pdb.post_mortem(einfo[2])
+            exit(1)
+
+    def show_location(self, lineno, column, show=3):
+        # Three lines ought to be enough
+        lower = lineno - show if lineno - show > 0 else 0
+        for i, line in enumerate(self.script.source.split('\n')[lower:lineno]):
+            print(lower + i + 1, line)
+        print(' ' * (column + len(str(lineno))), '^')
+
+    def show_operation(self):
+        print("%s:\n" % self.operation.capitalize())
+        getattr(self, 'show_' + self.operation)()
+
+    def show_completions(self):
+        for completion in self.completions:
+            print(completion.name)
+
+    # TODO: Support showing the location in other files
+
+    # TODO: Move this printing to the completion objects themselves
+    def show_usages(self):
+        for completion in self.completions:
+            print(completion.description)
+            if os.path.abspath(completion.module_path) == os.path.abspath(self.path):
+                self.show_location(completion.line, completion.column)
+
+    def show_call_signatures(self):
+        for completion in self.completions:
+            # This is too complicated to print. It really should be
+            # implemented in str() anyway.
+            print(completion)
+            # Can't print the location here because we don't have the module path
+
+    def show_goto_definitions(self):
+        for completion in self.completions:
+            print(completion.desc_with_module)
+            if os.path.abspath(completion.module_path) == os.path.abspath(self.path):
+                self.show_location(completion.line, completion.column)
+
+    show_goto_assignments = show_goto_definitions
+
+    def show_errors(self):
+        print(self.traceback)
+        print(("Error with running Script(...).{operation}() with\n"
+              "\tpath:   {path}\n"
+              "\tline:   {line}\n"
+              "\tcolumn: {column}").format(**self.__dict__))
+
+
+def main(arguments):
+    debugger = 'pdb' if arguments['--pdb'] else \
+               'ipdb' if arguments['--ipdb'] else \
+               'pudb' if arguments['--pudb'] else None
+    record = arguments['--record']
+
+    jedi.settings.use_filesystem_cache = arguments['--fs-cache']
+    if arguments['--debug']:
+        jedi.set_debug_function()
+
+    if arguments['redo'] or arguments['show']:
+        t = TestCase.from_cache(record)
+        if arguments['show']:
+            t.show_errors()
+        else:
+            t.run(debugger)
+    elif arguments['run']:
+            TestCase(
+                arguments['<operation>'], arguments['<path>'],
+                int(arguments['<line>']), int(arguments['<column>'])
+            ).run(debugger, print_result=True)
+    else:
+        for _ in range(int(arguments['--maxtries'])):
+            t = TestCase.generate(arguments['<path>'] or '.')
+            t.run(debugger, record)
+            print('.', end='')
+            sys.stdout.flush()
+        print()
+
+
+if __name__ == '__main__':
+    arguments = docopt(__doc__)
+    main(arguments)

test/absolute_import/local_module.py

@@ -0,0 +1,14 @@
+"""
+This is a module that imports the *standard library* unittest,
+despite there being a local "unittest" module. It specifies that it
+wants the stdlib one with the ``absolute_import`` __future__ import.
+
+The twisted equivalent of this module is ``twisted.trial._synctest``.
+"""
+from __future__ import absolute_import
+
+import unittest
+
+
+class Assertions(unittest.TestCase):
+    pass

test/absolute_import/unittest.py

@@ -0,0 +1,14 @@
+"""
+This is a module that shadows a builtin (intentionally).
+
+It imports a local module, which in turn imports stdlib unittest (the
+name shadowed by this module). If that is properly resolved, there's
+no problem. However, if jedi doesn't understand absolute_imports, it
+will get this module again, causing infinite recursion.
+"""
+from local_module import Assertions
+
+
+class TestCase(Assertions):
+    def test(self):
+        self.assertT

test/completion/__init__.py

@@ -0,0 +1,3 @@
+""" needed for some modules to test against packages. """
+
+some_variable = 1

test/completion/arrays.py

@@ -2,6 +2,7 @@
 # basic array lookups
 # -----------------
 
+
 #? int()
 [1,""][0]
 #? str()
@@ -10,6 +11,8 @@
 [1,""][2]
 #? int() str()
 [1,""][20]
+#? int() str()
+[1,""][str(hello)]
 
 a = list()
 #? list()
@@ -93,6 +96,28 @@ a4
 b4
 
 
+# -----------------
+# multiple assignments
+# -----------------
+a = b = 1
+#? int()
+a
+#? int()
+b
+
+(a, b) = (c, (e, f)) = ('2', (3, 4))
+#? str()
+a
+#? tuple()
+b
+#? str()
+c
+#? int()
+e
+#? int()
+f
+
+
 # -----------------
 # unnessecary braces
 # -----------------
@@ -117,6 +142,23 @@ u1
 #? int()
 a
 
+def a(): return ''
+#? str()
+(a)()
+#? str()
+(a)().replace()
+#? int()
+(tuple).index()
+#? int()
+(tuple)().index()
+
+class C():
+    def __init__(self):
+        self.a = (str()).upper()
+
+#? str()
+C().a
+
 # -----------------
 # imbalanced sides
 # -----------------
@@ -161,6 +203,31 @@ dic2[r'asdf']
 #? int() str()
 dic2['just_something']
 
+def f():
+    """ github #83 """
+    r = {}
+    r['status'] = (200, 'ok')
+    return r
+
+#? dict()
+f()
+
+# completion within dicts
+#? 9 ['str']
+{str: str}
+
+# iteration problem (detected with sith)
+d = dict({'a':''})
+def y(a):
+    return a
+#? 
+y(**d)
+
+# problem with more complicated casts
+dic = {str(key): ''}
+#? str()
+dic['']
+
 # -----------------
 # with variable as index
 # -----------------

test/completion/basic.py

@@ -9,6 +9,16 @@ int()
 int(str)
 
 
+# -----------------
+# should not complete
+# -----------------
+#? []
+.
+#? []
+str..
+#? []
+a(0):.
+
 # -----------------
 # if/else/elif
 # -----------------
@@ -72,6 +82,9 @@ for i in list([1,'']):
     #? int() str()
     i
 
+#? int() str()
+for x in [1,'']: x
+
 a = []
 b = [1.0,'']
 for i in b:
@@ -196,6 +209,9 @@ def a():
     """
     pass
 
+#? 
+# str literals in comment """ upper
+
 # -----------------
 # magic methods
 # -----------------
@@ -207,3 +223,45 @@ class B(): pass
 A.__init__
 #? ['__init__']
 B.__init__
+
+#? ['__init__']
+int().__init__
+
+# -----------------
+# comments
+# -----------------
+
+class A():
+    def __init__(self):
+        self.hello = {}  # comment shouldn't be a string
+#? dict()
+A().hello
+
+# -----------------
+# unicode
+# -----------------
+a = 'smörbröd'
+#? str()
+a
+xyz = 'smörbröd.py'
+if 1:
+    #? str()
+    xyz
+
+# -----------------
+# exceptions
+# -----------------
+try:
+    import math
+except ImportError as i_a:
+    #? ['i_a']
+    i_a
+    #? ImportError()
+    i_a
+try:
+    import math
+except ImportError, i_b:
+    #? ['i_b']
+    i_b
+    #? ImportError()
+    i_b

test/completion/classes.py

@@ -119,6 +119,13 @@ strs.second
 #? ['var_class']
 TestClass.var_class.var_class.var_class.var_class
 
+# operations (+, *, etc) shouldn't be InstanceElements - #246
+class A():
+    def __init__(self):
+        self.addition = 1 + 2
+#? int()
+A().addition
+
 # -----------------
 # inheritance
 # -----------------
@@ -241,192 +248,6 @@ class A():
 #? list()
 A().b()
 
-# -----------------
-# descriptors
-# -----------------
-class RevealAccess(object):
-    """
-    A data descriptor that sets and returns values
-    normally and prints a message logging their access.
-    """
-    def __init__(self, initval=None, name='var'):
-        self.val = initval
-        self.name = name
-
-    def __get__(self, obj, objtype):
-        print('Retrieving', self.name)
-        return self.val
-
-    def __set__(self, obj, val):
-        print('Updating', self.name)
-        self.val = val
-
-    def just_a_method(self):
-        pass
-
-class C(object):
-    x = RevealAccess(10, 'var "x"')
-    #? RevealAccess()
-    x
-    #? ['just_a_method']
-    x.just_a_method
-    y = 5.0
-    def __init__(self):
-        #? int()
-        self.x
-
-        #? []
-        self.just_a_method
-        #? []
-        C.just_a_method
-
-m = C()
-#? int()
-m.x
-#? float()
-m.y
-#? int()
-C.x
-
-#? []
-m.just_a_method
-#? []
-C.just_a_method
-
-# -----------------
-# properties
-# -----------------
-class B():
-    @property
-    def r(self):
-        return 1
-    @r.setter
-    def r(self, value):
-        return ''
-    def t(self):
-        return ''
-    p = property(t)
-
-#? []
-B().r()
-#? int()
-B().r
-
-#? str()
-B().p
-#? []
-B().p()
-
-class PropClass():
-    def __init__(self, a):
-        self.a = a
-    @property
-    def ret(self):
-        return self.a
-
-    @ret.setter
-    def ret(self, value):
-        return 1.0
-
-    def ret2(self):
-        return self.a
-    ret2 = property(ret2)
-
-    @property
-    def nested(self):
-        """ causes recusions in properties, should work """
-        return self.ret
-
-    @property
-    def nested2(self):
-        """ causes recusions in properties, should not work """
-        return self.nested2
-
-    @property
-    def join1(self):
-        """ mutual recusion """
-        return self.join2
-
-    @property
-    def join2(self):
-        """ mutual recusion """
-        return self.join1
-
-#? str()
-PropClass("").ret
-#? []
-PropClass().ret.
-
-#? str()
-PropClass("").ret2
-#? 
-PropClass().ret2
-
-#? int()
-PropClass(1).nested
-#? []
-PropClass().nested.
-
-#? 
-PropClass(1).nested2
-#? []
-PropClass().nested2.
-
-#? 
-PropClass(1).join1
-# -----------------
-# staticmethod/classmethod
-# -----------------
-
-class E(object):
-    a = ''
-    def __init__(self, a):
-        self.a = a
-
-    def f(x):
-        return x
-    f = staticmethod(f)
-
-    @staticmethod
-    def g(x):
-        return x
-
-    def s(cls, x):
-        return x
-    s = classmethod(s)
-
-    @classmethod
-    def t(cls, x):
-        return x
-
-    @classmethod
-    def u(cls, x):
-        return cls.a
-
-e = E(1)
-#? int()
-e.f(1)
-#? int()
-E.f(1)
-#? int()
-e.g(1)
-#? int()
-E.g(1)
-
-#? int()
-e.s(1)
-#? int()
-E.s(1)
-#? int()
-e.t(1)
-#? int()
-E.t(1)
-
-#? str()
-e.u(1)
-#? str()
-E.u(1)
-
 # -----------------
 # recursions
 # -----------------
@@ -488,6 +309,8 @@ getattr()
 getattr(str)
 #? 
 getattr(getattr, 1)
+#? 
+getattr(str, [])
 
 
 class Base():
@@ -523,3 +346,44 @@ class PrivateVar():
 PrivateVar().__var
 #? 
 PrivateVar().__var
+
+# -----------------
+# super
+# -----------------
+class Super(object):
+    a = 3
+
+class TestSuper(Super):
+    #? 
+    super()
+    def test(self):
+        #? Super()
+        super()
+        #? ['a']
+        super().a
+        if 1:
+            #? Super()
+            super()
+        def a():
+            #? 
+            super()
+
+
+# -----------------
+# if flow at class level
+# -----------------
+class TestX(object):
+    def normal_method(self):
+        return 1
+
+    if True:
+        def conditional_method(self):
+            var = self.normal_method()
+            #? int()
+            var
+            return 2
+
+    def other_method(self):
+        var = self.conditional_method()
+        #? int()
+        var

test/completion/decorators.py

@@ -76,7 +76,7 @@ exe[4]['d']
 
 
 # -----------------
-# class decorators
+# Decorator is a class
 # -----------------
 class Decorator(object):
     def __init__(self, func):
@@ -94,18 +94,65 @@ nothing("")[0]
 #? str()
 nothing("")[1]
 
+@Decorator
+def nothing(a,b,c):
+    return a,b,c
+
+class MethodDecoratorAsClass():
+    class_var = 3
+    @Decorator
+    def func_without_self(arg, arg2):
+        return arg, arg2
+
+    @Decorator
+    def func_with_self(self, arg):
+        return self.class_var
+
+#? int()
+MethodDecoratorAsClass().func_without_self('')[0]
+#? str()
+MethodDecoratorAsClass().func_without_self('')[1]
+#? 
+MethodDecoratorAsClass().func_with_self(1)
+
+
+class SelfVars():
+    """Init decorator problem as an instance, #247"""
+    @Decorator
+    def __init__(self):
+        """
+        init decorators should be ignored when looking up variables in the
+        class.
+        """
+        self.c = list
+
+    @Decorator
+    def shouldnt_expose_var(not_self):
+        """
+        Even though in real Python this shouldn't expose the variable, in this
+        case Jedi exposes the variable, because these kind of decorators are
+        normally descriptors, which SHOULD be exposed (at least 90%).
+        """
+        not_self.b = 1.0
+
+    def other_method(self):
+        #? float()
+        self.b
+        #? list
+        self.c
+
 # -----------------
-# not found decorators
+# not found decorators (are just ignored)
 # -----------------
 @not_found_decorator
 def just_a_func():
     return 1
 
-#? []
+#? int()
 just_a_func()
 
-#? []
-just_a_func.
+#? ['__closure__']
+just_a_func.__closure__
 
 
 class JustAClass:
@@ -113,14 +160,97 @@ class JustAClass:
     def a(self):
         return 1
 
-#? []
-JustAClass().a.
-#? []
-JustAClass().a()
-#? []
-JustAClass.a.
-#? []
+#? ['__closure__']
+JustAClass().a.__closure__
+#? int()
 JustAClass().a()
+#? ['__closure__']
+JustAClass.a.__closure__
+#? int()
+JustAClass.a()
+
+# -----------------
+# illegal decorators
+# -----------------
+
+class DecoratorWithoutCall():
+    def __init__(self, func):
+        self.func = func
+
+@DecoratorWithoutCall
+def f():
+    return 1
+
+# cannot be resolved - should be ignored
+@DecoratorWithoutCall(None)
+def g():
+    return 1
+
+#? 
+f()
+#? int()
+g()
+
+# -----------------
+# method decorators
+# -----------------
+
+def dec(f):
+    def wrapper(s):
+        return f(s)
+    return wrapper
+
+class MethodDecorators():
+    _class_var = 1
+    def __init__(self):
+        self._method_var = ''
+
+    @dec
+    def constant(self):
+        return 1.0
+
+    @dec
+    def class_var(self):
+        return self._class_var
+
+    @dec
+    def method_var(self):
+        return self._method_var
+
+#? float()
+MethodDecorators().constant()
+#? int()
+MethodDecorators().class_var()
+#? str()
+MethodDecorators().method_var()
+
+
+class Base():
+    @not_existing
+    def __init__(self):
+        pass
+    @not_existing
+    def b(self):
+        return ''
+    @dec
+    def c(self):
+        return 1
+
+class MethodDecoratorDoesntExist(Base):
+    """#272 github: combination of method decorators and super()"""
+    def a(self):
+        #? 
+        super().__init__()
+        #? str()
+        super().b()
+        #? int()
+        super().c()
+        #? float()
+        self.d()
+
+    @doesnt_exist
+    def d(self):
+        return 1.0
 
 # -----------------
 # others
@@ -148,5 +278,9 @@ follow_statement(1)
 
 # class decorators should just be ignored
 @should_ignore
-class A(): pass
+class A():
+    def ret(self):
+        return 1
 
+#? int()
+A().ret()

test/completion/definition.py

@@ -0,0 +1,65 @@
+"""
+Fallback to callee definition when definition not found.
+- https://github.com/davidhalter/jedi/issues/131
+- https://github.com/davidhalter/jedi/pull/149
+"""
+
+"""Parenthesis closed at next line."""
+
+#? isinstance
+isinstance(
+)
+
+#? isinstance
+isinstance( 
+)
+
+#? isinstance
+isinstance(None,
+)
+
+#? isinstance
+isinstance(None, 
+)
+
+"""Parenthesis closed at same line."""
+
+# Note: len('isinstance(') == 11
+#? 11 isinstance
+isinstance()
+
+# Note: len('isinstance(None,') == 16
+##? 16 isinstance
+isinstance(None,)
+
+# Note: len('isinstance(None,') == 16
+##? 16 isinstance
+isinstance(None, )
+
+# Note: len('isinstance(None, ') == 17
+##? 17 isinstance
+isinstance(None, )
+
+# Note: len('isinstance( ') == 12
+##? 12 isinstance
+isinstance( )
+
+"""Unclosed parenthesis."""
+
+#? isinstance
+isinstance(
+
+def x(): pass  # acts like EOF
+
+##? isinstance
+isinstance( 
+
+def x(): pass  # acts like EOF
+
+#? isinstance
+isinstance(None,
+
+def x(): pass  # acts like EOF
+
+##? isinstance
+isinstance(None, 

test/completion/descriptors.py

@@ -0,0 +1,182 @@
+class RevealAccess(object):
+    """
+    A data descriptor that sets and returns values
+    normally and prints a message logging their access.
+    """
+    def __init__(self, initval=None, name='var'):
+        self.val = initval
+        self.name = name
+
+    def __get__(self, obj, objtype):
+        print('Retrieving', self.name)
+        return self.val
+
+    def __set__(self, obj, val):
+        print('Updating', self.name)
+        self.val = val
+
+    def just_a_method(self):
+        pass
+
+class C(object):
+    x = RevealAccess(10, 'var "x"')
+    #? RevealAccess()
+    x
+    #? ['just_a_method']
+    x.just_a_method
+    y = 5.0
+    def __init__(self):
+        #? int()
+        self.x
+
+        #? []
+        self.just_a_method
+        #? []
+        C.just_a_method
+
+m = C()
+#? int()
+m.x
+#? float()
+m.y
+#? int()
+C.x
+
+#? []
+m.just_a_method
+#? []
+C.just_a_method
+
+# -----------------
+# properties
+# -----------------
+class B():
+    @property
+    def r(self):
+        return 1
+    @r.setter
+    def r(self, value):
+        return ''
+    def t(self):
+        return ''
+    p = property(t)
+
+#? []
+B().r()
+#? int()
+B().r
+
+#? str()
+B().p
+#? []
+B().p()
+
+class PropClass():
+    def __init__(self, a):
+        self.a = a
+    @property
+    def ret(self):
+        return self.a
+
+    @ret.setter
+    def ret(self, value):
+        return 1.0
+
+    def ret2(self):
+        return self.a
+    ret2 = property(ret2)
+
+    @property
+    def nested(self):
+        """ causes recusions in properties, should work """
+        return self.ret
+
+    @property
+    def nested2(self):
+        """ causes recusions in properties, should not work """
+        return self.nested2
+
+    @property
+    def join1(self):
+        """ mutual recusion """
+        return self.join2
+
+    @property
+    def join2(self):
+        """ mutual recusion """
+        return self.join1
+
+#? str()
+PropClass("").ret
+#? []
+PropClass().ret.
+
+#? str()
+PropClass("").ret2
+#? 
+PropClass().ret2
+
+#? int()
+PropClass(1).nested
+#? []
+PropClass().nested.
+
+#? 
+PropClass(1).nested2
+#? []
+PropClass().nested2.
+
+#? 
+PropClass(1).join1
+# -----------------
+# staticmethod/classmethod
+# -----------------
+
+class E(object):
+    a = ''
+    def __init__(self, a):
+        self.a = a
+
+    def f(x):
+        return x
+    f = staticmethod(f)
+
+    @staticmethod
+    def g(x):
+        return x
+
+    def s(cls, x):
+        return x
+    s = classmethod(s)
+
+    @classmethod
+    def t(cls, x):
+        return x
+
+    @classmethod
+    def u(cls, x):
+        return cls.a
+
+e = E(1)
+#? int()
+e.f(1)
+#? int()
+E.f(1)
+#? int()
+e.g(1)
+#? int()
+E.g(1)
+
+#? int()
+e.s(1)
+#? int()
+E.s(1)
+#? int()
+e.t(1)
+#? int()
+E.t(1)
+
+#? str()
+e.u(1)
+#? str()
+E.u(1)

test/completion/docstring.py

@@ -0,0 +1,121 @@
+""" Test docstrings in functions and classes, which are used to infer types """
+
+# -----------------
+# sphinx style
+# -----------------
+def f(a, b, c, d):
+    """ asdfasdf
+    :param a: blablabla
+    :type a: str
+    :type b: (str, int)
+    :type c: threading.Thread
+    :type d: :class:`threading.Thread`
+    :rtype: dict
+    """
+    #? str()
+    a
+    #? str()
+    b[0]
+    #? int()
+    b[1]
+    #? ['join']
+    c.join
+    #? ['join']
+    d.join
+
+#? dict()
+f()
+
+# wrong declarations
+def f(a, b):
+    """
+    :param a: Forgot type declaration
+    :type a:
+    :param b: Just something
+    :type b: ``
+    :rtype: 
+    """
+    #? 
+    a
+    #? 
+    b
+
+#? 
+f()
+
+# -----------------
+# epydoc style
+# -----------------
+def e(a, b):
+    """ asdfasdf
+    @type a: str
+    @param a: blablabla
+    @type b: (str, int)
+    @param b: blablah
+    @rtype: list
+    """
+    #? str()
+    a
+    #? str()
+    b[0]
+
+    #? int()
+    b[1]
+
+#? list()
+e()
+
+
+# Returns with param type only
+def rparam(a,b):
+    """
+    @type a: str
+    """
+    return a
+
+#? str()
+rparam()
+
+
+# Composite types
+def composite():
+    """
+    @rtype: (str, int, dict)
+    """
+
+x, y, z = composite()
+#? str()
+x
+#? int()
+y
+#? dict()
+z
+
+
+# Both docstring and calculated return type
+def both():
+    """
+    @rtype: str
+    """
+    return 23
+
+#? str(), int()
+both()
+
+class Test(object):
+    def __init__(self):
+        self.teststr = ""
+    """
+    # jedi issue #210
+    """
+    def test(self):
+        #? ['teststr']
+        self.teststr
+
+# -----------------
+# statement docstrings
+# -----------------
+d = ''
+""" bsdf """
+#? str()
+d.upper()

test/completion/dynamic.py

@@ -63,6 +63,11 @@ def func(c=1):
 
 func(1.0)
 
+# Needs to be here, because in this case func is an import -> shouldn't lead to
+# exceptions.
+import sys as func
+func.sys
+
 # -----------------
 # classes
 # -----------------

test/completion/functions.py

@@ -46,6 +46,15 @@ def multi_line_func(a, # comment blabla
 #? str()
 multi_line_func(1,'')
 
+# nothing after comma
+def asdf(a):
+    return a
+
+x = asdf(a=1,
+    )
+#? int()
+x
+
 # -----------------
 # double execution
 # -----------------
@@ -109,10 +118,10 @@ def func(a=1, b=''):
     return a, b
 
 exe = func(b=list, a=tuple)
-#? tuple()
+#? tuple
 exe[0]
 
-#? list()
+#? list
 exe[1]
 
 # -----------------
@@ -146,15 +155,29 @@ def a():
 # -----------------
 
 def args_func(*args):
+    #? tuple()
     return args
 
 exe = args_func(1, "")
 #? int()
 exe[0]
-
 #? str()
 exe[1]
 
+# illegal args (TypeError)
+#? 
+args_func(*1)[0]
+# iterator
+#? int()
+args_func(*iter([1]))[0]
+
+# different types
+e = args_func(*[1+"", {}])
+#? int() str()
+e[0]
+#? dict()
+e[1]
+
 _list = [1,""]
 exe2 = args_func(_list)[0]
 
@@ -181,6 +204,9 @@ exe[1][1]
 # ** kwargs
 # -----------------
 def kwargs_func(**kwargs):
+    #? ['keys']
+    kwargs.keys
+    #? dict()
     return kwargs
 
 exe = kwargs_func(a=3,b=4.0)
@@ -202,6 +228,12 @@ exe2['a']
 # *args / ** kwargs
 # -----------------
 
+def func_without_call(*args, **kwargs):
+    #? tuple()
+    args
+    #? dict()
+    kwargs
+
 def fu(a=1, b="", *args, **kwargs):
     return a, b, args, kwargs
 
@@ -354,3 +386,66 @@ annot_ret('')
 def a(): pass
 #? ['__closure__']
 a.__closure__
+
+
+# -----------------
+# lambdas
+# -----------------
+a = lambda: 3
+#? int()
+a()
+
+x = []
+a = lambda x: x
+#? int()
+a(0)
+
+#? float()
+(lambda x: x)(3.0)
+
+arg_l = lambda x, y: y, x
+#? float()
+arg_l[0]('', 1.0)
+#? list()
+arg_l[1]
+
+arg_l = lambda x, y: (y, x)
+args = 1,""
+result = arg_l(*args)
+#? tuple()
+result
+#? str()
+result[0]
+#? int()
+result[1]
+
+def with_lambda(callable_lambda, *args, **kwargs):
+    return callable_lambda(1, *args, **kwargs)
+
+#? int()
+with_lambda(arg_l, 1.0)[1]
+#? float()
+with_lambda(arg_l, 1.0)[0]
+#? float()
+with_lambda(arg_l, y=1.0)[0]
+#? int()
+with_lambda(lambda x: x)
+#? float()
+with_lambda(lambda x, y: y, y=1.0)
+
+arg_func = lambda *args, **kwargs: (args[0], kwargs['a'])
+#? int()
+arg_func(1, 2, a='', b=10)[0]
+#? list()
+arg_func(1, 2, a=[], b=10)[1]
+
+# magic method
+a = lambda: 3
+#? ['__closure__']
+a.__closure__
+
+class C():
+    def __init__(self):
+        self.a = lambda: 1
+#? int()
+C().a()

test/completion/goto.py

@@ -1,19 +1,25 @@
-# goto command test are a different in syntax
+# goto_assignments command tests are different in syntax
 
 definition = 3
-##! 0 ['a=definition']
+#! 0 ['a = definition']
 a = definition
 
 #! []
 b
-#! ['a=definition']
+#! ['a = definition']
 a
 
 b = a
 c = b
-#! ['c=b']
+#! ['c = b']
 c
 
+cd = 1
+#! 1 ['cd = c']
+cd = c
+#! 0 ['cd = e']
+cd = e
+
 #! ['module math']
 import math
 #! ['import math']
@@ -21,12 +27,12 @@ math
 
 #! ['import math']
 b = math
-#! ['b=math']
+#! ['b = math']
 b
 
 class C(object):
     def b(self):
-        #! ['b=math']
+        #! ['b = math']
         b
         #! ['def b']
         self.b
@@ -39,7 +45,7 @@ class C(object):
     #! ['def b']
     b
 
-#! ['b=math']
+#! ['b = math']
 b
 
 #! ['def b']
@@ -57,9 +63,9 @@ D.b
 #! ['def b']
 D().b
 
-#! 0 ['D=C']
+#! 0 ['D = C']
 D().b
-#! 0 ['D=C']
+#! 0 ['D = C']
 D().b
 
 def c():
@@ -76,39 +82,45 @@ c()
 
 #! ['module import_tree']
 import import_tree
-#! ['a=""']
+#! ["a = ''"]
 import_tree.a
 
 #! ['module mod1']
 import import_tree.mod1
-#! ['a=1']
+#! ['a = 1']
 import_tree.mod1.a
 
 #! ['module pkg']
 import import_tree.pkg
-#! ['a=list']
+#! ['a = list']
 import_tree.pkg.a
 
 #! ['module mod1']
 import import_tree.pkg.mod1
-#! ['a=1.0']
+#! ['a = 1.0']
 import_tree.pkg.mod1.a
-#! ['a=""']
+#! ["a = ''"]
 import_tree.a
 
 #! ['module mod1']
 from import_tree.pkg import mod1
-#! ['a=1.0']
+#! ['a = 1.0']
 mod1.a
 
 #! ['module mod1']
 from import_tree import mod1
-#! ['a=1']
+#! ['a = 1']
 mod1.a
 
-#! ['a=1.0']
+#! ['a = 1.0']
 from import_tree.pkg.mod1 import a
 
+#! ['import os']
+from .imports import os
+
+#! ['some_variable = 1']
+from . import some_variable
+
 # -----------------
 # anonymous classes
 # -----------------
@@ -121,11 +133,41 @@ def func():
 #! 8 ['def b']
 func().b()
 
-# -----------------    
+# -----------------
 # on itself
-# -----------------    
+# -----------------
 
 #! 7 ['class ClassDef']
 class ClassDef():
     """ abc """
     pass
+
+# -----------------
+# params
+# -----------------
+
+param = ClassDef
+#! 8 ['param']
+def ab1(param): pass
+#! 9 ['param']
+def ab2(param): pass
+#! 11 ['param = ClassDef']
+def ab3(a=param): pass
+
+ab1(ClassDef);ab2(ClassDef);ab3(ClassDef)
+
+# -----------------
+# for loops
+# -----------------
+
+for i in range(1):
+    #! ['for i in range(1):    i']
+    i
+
+for key, value in [(1,2)]:
+    #! ['for key,value in [(1...']
+    key
+
+for i in []:
+    #! ['for i in []:    i']
+    i

test/completion/import_tree/mod1.py

@@ -1 +1,2 @@
 a = 1
+from import_tree.random import a as c

test/completion/import_tree/mod2.py

@@ -0,0 +1 @@
+from . import mod1 as fake

test/completion/import_tree/recurse_class1.py

@@ -0,0 +1,5 @@
+import recurse_class2
+
+class C(recurse_class2.C):
+    def a(self):
+        pass

test/completion/import_tree/recurse_class2.py

@@ -0,0 +1,4 @@
+import recurse_class1
+
+class C(recurse_class1.C):
+    pass

test/completion/imports.py

@@ -56,6 +56,53 @@ def scope_nested():
     #? set
     import_tree.random.a
 
+def scope_nested2():
+    """Multiple modules should be indexable, if imported"""
+    import import_tree.mod1
+    import import_tree.pkg
+    #? ['mod1']
+    import_tree.mod1
+    #? ['pkg']
+    import_tree.pkg
+    #? []
+    import_tree.rename1
+
+def from_names():
+    #? ['mod1']
+    from import_tree.pkg.
+    #? ['path']
+    from os.
+
+def builtin_test():
+    #? ['math']
+    import math
+
+def scope_from_import_variable():
+    """
+    All of them shouldn't work, because "fake" imports don't work in python
+    without the use of ``sys.modules`` modifications (e.g. ``os.path`` see also
+    github issue #213 for clarification.
+    """
+    #? 
+    from import_tree.mod2.fake import a
+    #? 
+    from import_tree.mod2.fake import c
+
+    #? 
+    a
+    #? 
+    c
+
+def scope_from_import_variable_with_parenthesis():
+    from import_tree.mod2.fake import (
+        a, c
+    )
+
+    #? 
+    a
+    #? 
+    c
+
 # -----------------
 # std lib modules
 # -----------------
@@ -72,6 +119,16 @@ import os
 #? ['dirname']
 os.path.dirname
 
+#? os.path.join
+from os.path import join
+
+from os.path import (
+    expanduser
+)
+
+#? os.path.expanduser
+expanduser
+
 from itertools import (tee,
                        islice)
 #? ['islice']
@@ -152,11 +209,11 @@ from .......import_tree import mod1
 #? 
 mod1.a
 
-from .. import run
+from .. import helpers
 #? int()
-run.tests_fail
+helpers.sample_int
 
-from ..run import tests_fail as f
+from ..helpers import sample_int as f
 #? int()
 f
 
@@ -175,6 +232,13 @@ mod1.
 #? str()
 imp_tree.a
 
+#? ['some_variable']
+from . import some_variable
+#? ['arrays']
+from . import arrays
+#? []
+from . import import_tree as ren
+
 
 # -----------------
 # special positions -> edge cases
@@ -189,7 +253,9 @@ import datetime.
 #? []
 import datetime.date
 
-#? 18 ['mod1', 'random', 'pkg', 'rename1', 'rename2']
+#? 18 ['import']
+from import_tree. import pkg
+#? 17 ['mod1', 'mod2', 'random', 'pkg', 'rename1', 'rename2', 'recurse_class1', 'recurse_class2']
 from import_tree. import pkg
 
 #? 18 ['pkg']
@@ -208,3 +274,32 @@ from not_a_module import
 # self import
 # this can cause recursions
 from imports import *
+
+#137
+import json
+#? 23 json.dump
+from json import load, dump
+#? 17 json.load
+from json import load, dump
+# without the from clause:
+import json, datetime
+#? 7 json
+import json, datetime
+#? 13 datetime
+import json, datetime
+
+# -----------------
+# packages
+# -----------------
+
+from import_tree.mod1 import c
+#? set
+c
+
+from import_tree import recurse_class1
+
+#? ['a']
+recurse_class1.C.a
+# github #239 RecursionError
+#? ['a']
+recurse_class1.C().a

test/completion/invalid.py

@@ -6,9 +6,22 @@ Basically this file could change depending on the current implementation. But
 there should never be any errors.
 """
 
+# wait until keywords are out of definitions (pydoc function).
+##? 5 
+'s'()
+
 #? ['upper']
 str()).upper
 
+# -----------------
+# funcs
+# -----------------
+def asdf(a or b): # multiple param names
+    return a
+
+#? int()
+asdf(2)
+
 from a import (b
 def blub():
     return 0
@@ -42,6 +55,22 @@ def normalfunc():
 #? int()
 normalfunc()
 
+# dots in param
+def f(seq1...=None):
+    return seq1
+#? int()
+f(1)
+
+@
+def test_empty_decorator():
+    return 1
+
+#? int()
+test_empty_decorator()
+
+# -----------------
+# flows
+# -----------------
 
 # first part not complete (raised errors)
 if a
@@ -77,6 +106,10 @@ for_local
 for_local
 
 
+# -----------------
+# list comprehensions
+# -----------------
+
 a2 = [for a2 in [0]]
 #? 
 a2[0]
@@ -101,16 +134,23 @@ a[0]
 
 a = [a for a in [1,2]
 def break(): pass
-#? list()
+#? int()
 a[0]
 
 #? []
 int()).
 
-def asdf(a or b): # multiple param names
-    return a
+# -----------------
+# keywords
+# -----------------
 
-#? int()
-asdf(2)
+#! []
+as
+
+def empty_assert():
+    x = 3
+    assert
+    #? int()
+    x
 
 import datetime as 

test/completion/named_param.py

@@ -1,5 +1,6 @@
 """ named params:
 >>> def a(abc): pass
+...
 >>> a(abc=3)  # <- this stuff
 """
 

test/completion/renaming.py

@@ -1,99 +0,0 @@
-"""
-Renaming tests. This means search for related names.
-I always leave a little bit of space to add room for additions, because the
-results always contain position informations.
-"""
-#< 4 (7,4), (10,0), (12,0)
-def abc(): pass
-
-#< 0 (7,4), (10,0), (12,0)
-abc.d.a.bsaasd.abc.d
-
-abc
-
-
-
-
-
-
-
-abc = 
-
-#< (20,0), (23,0)
-abc
-
-
-
-Abc = 3
-
-#< 6 (30,6), (32,4), (35,8), (47,0)
-class Abc():
-    #< (30,6), (32,4), (35,8), (47,0)
-    Abc
-
-    def Abc(self):
-        Abc; self.c = 3
-
-    #< 17 (38,16), (40,8)
-    def a(self, Abc):
-        #< 10 (38,16), (40,8)
-        Abc
-
-    #< 19 (43,18), (45,8)
-    def self_test(self):
-        #< 12 (43,18), (45,8)
-        self.b
-
-Abc.d.Abc
-
-
-
-
-
-
-#< 4 (55,4), (59,1)
-def blub():
-
-
-#< (55,4), (59,1)
-@blub
-def a(): pass
-
-
-
-#< (65,7), (68,0)
-import module_not_exists
-
-#< (65,7), (68,0)
-module_not_exists
-
-
-
-
-
-#< (1,0), (75,24), (78,0), (81,17), (4,5), (85,17), (88,17)
-from import_tree import rename1
-
-#< (78,8), (3,0), (4,20), (6,0), (81,32), (85,32), (82,0)
-rename1.abc
-
-#< (78,8), (3,0), (4,20), (6,0), (81,32), (85,32), (82,0)
-from import_tree.rename1 import abc
-abc
-
-#< 20 (1,0), (4,5), (75,24), (78,0), (81,17), (85,17), (88,17)
-from import_tree.rename1 import abc
-
-#< (88, 32),
-from import_tree.rename1 import not_existing
-
-
-
-
-response = 5
-#< 0 (95, 0), (96, 0), (97, 0), (99, 0)
-response = HttpResponse(mimetype='application/pdf')
-response['Content-Disposition'] = 'attachment; filename=%s.pdf' % id
-response.write(pdf)
-#< (95, 0), (96, 0), (97, 0), (99, 0)
-response

test/completion/std.py

@@ -67,11 +67,25 @@ def example(a):
 #? str()
 example('')
 
+
 # -----------------
-# various regression tests
+# sqlite3 (#84)
 # -----------------
 
-#62
-import threading
-#? ['_Verbose', '_VERBOSE']
-threading._Verbose
+import sqlite3
+#? sqlite3.Connection()
+con = sqlite3.connect()
+#? sqlite3.Cursor()
+c = con.cursor()
+#? sqlite3.Row()
+row = c.fetchall()[0]
+#? str()
+row.keys()[0]
+
+def huhu(db):
+    """
+        :type db: sqlite3.Connection
+        :param db: the db connection
+    """
+    #? sqlite3.Connection()
+    db

test/completion/sys_path.py

@@ -12,7 +12,7 @@ sys.path.append(sys.path[1] + '/thirdparty')
 # syntax err
 sys.path.append('a' +* '/thirdparty')
 
-#? ['evaluate']
+#? ['evaluate', 'evaluate_representation']
 import evaluate
 
 #? ['goto']

test/completion/thirdparty/jedi_.py

@@ -1,7 +1,7 @@
 
 from jedi import functions, evaluate, parsing
 
-el = functions.complete()[0]
+el = functions.completions()[0]
 #? ['description']
 el.description
 
@@ -10,8 +10,7 @@ el.description
 
 
 scopes, path, dot, like = \
-    functions.prepare_goto(source, row, column,
-                            source_path, True)
+    api._prepare_goto(source, row, column, path, True)
 
 # has problems with that (sometimes) very deep nesting.
 #? set()

test/completion/usages.py

@@ -0,0 +1,233 @@
+"""
+Renaming tests. This means search for usages.
+I always leave a little bit of space to add room for additions, because the
+results always contain position informations.
+"""
+#< 4 (0,4), (3,0), (5,0)
+def abc(): pass
+
+#< 0 (-3,4), (0,0), (2,0)
+abc.d.a.bsaasd.abc.d
+
+abc
+
+abc = 
+
+#< (-3,0), (0,0)
+abc
+
+
+Abc = 3
+
+#< 6 (0,6), (2,4), (5,8), (17,0)
+class Abc():
+    #< (-2,6), (0,4), (3,8), (15,0)
+    Abc
+
+    def Abc(self):
+        Abc; self.c = 3
+
+    #< 17 (0,16), (2,8)
+    def a(self, Abc):
+        #< 10 (-2,16), (0,8)
+        Abc
+
+    #< 19 (0,18), (2,8)
+    def self_test(self):
+        #< 12 (-2,18), (0,8)
+        self.b
+
+Abc.d.Abc
+
+
+#< 4 (0,4), (4,1)
+def blub():
+
+
+#< (-4,4), (0,1)
+@blub
+def a(): pass
+
+
+#< 0 (0,0), (1,0)
+set_object_var = object()
+set_object_var.var = 1
+
+
+response = 5
+#< 0 (0,0), (1,0), (2,0), (4,0)
+response = HttpResponse(mimetype='application/pdf')
+response['Content-Disposition'] = 'attachment; filename=%s.pdf' % id
+response.write(pdf)
+#< (-4,0), (-3,0), (-2,0), (0,0)
+response
+
+
+# -----------------
+# imports
+# -----------------
+#< (0,7), (3,0)
+import module_not_exists
+
+#< (-3,7), (0,0)
+module_not_exists
+
+
+#< ('rename1', 1,0), (0,24), (3,0), (6,17), ('rename2', 4,5), (10,17), (13,17)
+from import_tree import rename1
+
+#< (0,8), ('rename1',3,0), ('rename2',4,20), ('rename2',6,0), (3,32), (7,32), (4,0)
+rename1.abc
+
+#< (-3,8), ('rename1', 3,0), ('rename2', 4,20), ('rename2', 6,0), (0,32), (4,32), (1,0)
+from import_tree.rename1 import abc
+abc
+
+#< 20 ('rename1', 1,0), ('rename2', 4,5), (-10,24), (-7,0), (-4,17), (0,17), (3,17)
+from import_tree.rename1 import abc
+
+#< (0, 32),
+from import_tree.rename1 import not_existing
+
+# shouldn't work
+#< 
+from not_existing import *
+
+# -----------------
+# classes
+# -----------------
+
+class TestMethods(object):
+    #< 8 (0,8), (2,13)
+    def a_method(self):
+        #< 13 (-2,8), (0,13)
+        self.a_method()
+        #< 13 (2,8), (0,13), (3,13)
+        self.b_method()
+
+    def b_method(self):
+        self.b_method
+
+
+class TestClassVar(object):
+    #< 4 (0,4), (5,13), (7,21)
+    class_v = 1
+    def a(self):
+        class_v = 1
+
+        #< (-5,4), (0,13), (2,21)
+        self.class_v
+        #< (-7,4), (-2,13), (0,21)
+        TestClassVar.class_v
+        #< (0,8), (-7, 8)
+        class_v
+
+class TestInstanceVar():
+    def a(self):
+        #< 13 (4,13), (0,13)
+        self._instance_var = 3
+
+    def b(self):
+        #< (-4,13), (0,13)
+        self._instance_var
+
+
+class NestedClass():
+    def __getattr__(self, name):
+        return self
+
+# Shouldn't find a definition, because there's no name defined (used ``getattr``).
+
+#< (0, 14),
+NestedClass().instance
+
+
+# -----------------
+# inheritance
+# -----------------
+class Super(object):
+    #< 4 (0,4), (23,18), (25,13)
+    base_class = 1
+    #< 4 (0,4),
+    class_var = 1
+
+    #< 8 (0,8),
+    def base_method(self):
+        #< 13 (0,13), (20,13)
+        self.base_var = 1
+        #< 13 (0,13), (24,13), (29,13)
+        self.instance_var = 1
+
+    #< 8 (0,8),
+    def just_a_method(self): pass
+
+
+#< 20 (0,16), (-18,6)
+class TestClass(Super):
+    #< 4 (0,4),
+    class_var = 1
+
+    def x_method(self):
+
+        #< (0,18), (2,13), (-23,4)
+        TestClass.base_class
+        #< (-2,18), (0,13), (-25,4)
+        self.base_class
+        #< (-20,13), (0,13)
+        self.base_var
+        #< 
+        TestClass.base_var
+
+
+        #< 13 (5,13), (0,13)
+        self.instance_var = 3
+
+    #< 9 (0,8), 
+    def just_a_method(self):
+        #< (-5,13), (0,13), (-29,13)
+        self.instance_var
+
+
+# -----------------
+# properties
+# -----------------
+class TestProperty:
+
+    @property
+    #< 10 (0,8), (5,13)
+    def prop(self):
+        return 1
+
+    def a(self):
+        #< 13 (-5,8), (0,13)
+        self.prop
+
+    @property
+    #< 13 (0,8), (4,5)
+    def rw_prop(self):
+        return self._rw_prop
+
+    #< 8 (-4,8), (0,5)
+    @rw_prop.setter
+    #< 8 (0,8), (5,13)
+    def rw_prop(self, value):
+        self._rw_prop = value
+
+    def b(self):
+        #< 13 (-5,8), (0,13)
+        self.rw_prop
+
+# -----------------
+# *args, **kwargs
+# -----------------
+#< 11 (1,11), (0,8)
+def f(**kwargs):
+    return kwargs
+
+
+# -----------------
+# No result
+# -----------------
+if isinstance(j, int):
+    #< 
+    j

test/conftest.py

@@ -0,0 +1,99 @@
+import os
+import shutil
+import tempfile
+
+import pytest
+
+from . import helpers
+from . import run
+from . import refactor
+
+
+def pytest_addoption(parser):
+    parser.addoption(
+        "--integration-case-dir",
+        default=os.path.join(helpers.test_dir, 'completion'),
+        help="Directory in which integration test case files locate.")
+    parser.addoption(
+        "--refactor-case-dir",
+        default=os.path.join(helpers.test_dir, 'refactor'),
+        help="Directory in which refactoring test case files locate.")
+    parser.addoption(
+        "--test-files", "-T", default=[], action='append',
+        help=(
+            "Specify test files using FILE_NAME[:LINE[,LINE[,...]]]. "
+            "For example: -T generators.py:10,13,19. "
+            "Note that you can use -m to specify the test case by id."))
+    parser.addoption(
+        "--thirdparty", action='store_true',
+        help="Include integration tests that requires third party modules.")
+
+
+def parse_test_files_option(opt):
+    """
+    Parse option passed to --test-files into a key-value pair.
+
+    >>> parse_test_files_option('generators.py:10,13,19')
+    ('generators.py', [10, 13, 19])
+    """
+    opt = str(opt)
+    if ':' in opt:
+        (f_name, rest) = opt.split(':', 1)
+        return (f_name, list(map(int, rest.split(','))))
+    else:
+        return (opt, [])
+
+
+def pytest_generate_tests(metafunc):
+    """
+    :type metafunc: _pytest.python.Metafunc
+    """
+    test_files = dict(map(parse_test_files_option,
+                          metafunc.config.option.test_files))
+    if 'case' in metafunc.fixturenames:
+        base_dir = metafunc.config.option.integration_case_dir
+        thirdparty = metafunc.config.option.thirdparty
+        cases = list(run.collect_dir_tests(base_dir, test_files))
+        if thirdparty:
+            cases.extend(run.collect_dir_tests(
+                os.path.join(base_dir, 'thirdparty'), test_files, True))
+        metafunc.parametrize('case', cases)
+    if 'refactor_case' in metafunc.fixturenames:
+        base_dir = metafunc.config.option.refactor_case_dir
+        metafunc.parametrize(
+            'refactor_case',
+            refactor.collect_dir_tests(base_dir, test_files))
+
+
+@pytest.fixture()
+def isolated_jedi_cache(monkeypatch, tmpdir):
+    """
+    Set `jedi.settings.cache_directory` to a temporary directory during test.
+
+    Same as `clean_jedi_cache`, but create the temporary directory for
+    each test case (scope='function').
+    """
+    from jedi import settings
+    monkeypatch.setattr(settings, 'cache_directory', str(tmpdir))
+
+
+@pytest.fixture(scope='session')
+def clean_jedi_cache(request):
+    """
+    Set `jedi.settings.cache_directory` to a temporary directory during test.
+
+    Note that you can't use built-in `tmpdir` and `monkeypatch`
+    fixture here because their scope is 'function', which is not used
+    in 'session' scope fixture.
+
+    This fixture is activated in ../pytest.ini.
+    """
+    from jedi import settings
+    old = settings.cache_directory
+    tmp = tempfile.mkdtemp(prefix='jedi-test-')
+    settings.cache_directory = tmp
+
+    @request.addfinalizer
+    def restore():
+        settings.cache_directory = old
+        shutil.rmtree(tmp)