Write the CHANGELOG for 0.15.1

This is in the latest 0.15, and when forwarding path completions to
jedi, print a lot of stuff on the screen.

.coveragerc

@@ -1,19 +1,13 @@
 [run]
 omit =
     jedi/_compatibility.py
-    jedi/evaluate/site.py
+    jedi/evaluate/compiled/subprocess/__main__.py
+    jedi/__main__.py
+    # For now this is not being used.
+    jedi/refactoring.py
 
 [report]
 # Regexes for lines to exclude from consideration
 exclude_lines =
     # Don't complain about missing debug-only code:
     def __repr__
-    if self\.debug
-
-    # Don't complain if tests don't hit defensive assertion code:
-    raise AssertionError
-    raise NotImplementedError
-
-    # Don't complain if non-runnable code isn't run:
-    if 0:
-    if __name__ == .__main__.:

.gitignore

@@ -5,9 +5,11 @@
 .tox
 .coveralls.yml
 .coverage
+.idea
 /build/
 /docs/_build/
 /dist/
 jedi.egg-info/
 record.json
 /.cache/
+/.pytest_cache

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "jedi/third_party/typeshed"]
+	path = jedi/third_party/typeshed
+	url = https://github.com/davidhalter/typeshed.git

.travis.yml

@@ -1,30 +1,68 @@
+dist: xenial
 language: python
-sudo: false
 python:
-  - 2.6
   - 2.7
-  - 3.3
   - 3.4
   - 3.5
   - 3.6
-  - pypy
+  - 3.7
+
+env:
+  - JEDI_TEST_ENVIRONMENT=27
+  - JEDI_TEST_ENVIRONMENT=34
+  - JEDI_TEST_ENVIRONMENT=35
+  - JEDI_TEST_ENVIRONMENT=36
+  - JEDI_TEST_ENVIRONMENT=37
+
 matrix:
-  allow_failures:
-    - python: pypy
-    - env: TOXENV=cov
-    - env: TOXENV=sith
   include:
-    - python: 3.5
-      env: TOXENV=cov
-    - python: 3.5
-      env: TOXENV=sith
+    - python: 3.6
+      env:
+        - TOXENV=cov
+        - JEDI_TEST_ENVIRONMENT=36
+    # For now ignore pypy, there are so many issues that we don't really need
+    # to run it.
+    #- python: pypy
+    - python: 3.8-dev
+      env:
+        - JEDI_TEST_ENVIRONMENT=38
 install:
     - pip install --quiet tox-travis
 script:
+    - |
+      # Setup/install Python for $JEDI_TEST_ENVIRONMENT.
+      set -ex
+      test_env_version=${JEDI_TEST_ENVIRONMENT:0:1}.${JEDI_TEST_ENVIRONMENT:1:1}
+      if [ "$TRAVIS_PYTHON_VERSION" != "$test_env_version" ]; then
+        python_bin=python$test_env_version
+        python_path="$(which $python_bin || true)"
+        if [ -z "$python_path" ]; then
+          # Only required for JEDI_TEST_ENVIRONMENT=34.
+          download_name=python-$test_env_version
+          wget https://s3.amazonaws.com/travis-python-archives/binaries/ubuntu/16.04/x86_64/$download_name.tar.bz2
+          sudo tar xjf $download_name.tar.bz2 --directory / opt/python
+          ln -s "/opt/python/${test_env_version}/bin/python" /home/travis/bin/$python_bin
+        elif [ "${python_path#/opt/pyenv/shims}" != "$python_path" ]; then
+          # Activate pyenv version (required with JEDI_TEST_ENVIRONMENT=36).
+          pyenv_bin="$(pyenv whence --path "$python_bin" | head -n1)"
+          ln -s "$pyenv_bin" /home/travis/bin/$python_bin
+        fi
+        $python_bin --version
+        python_ver=$($python_bin -c 'import sys; print("%d%d" % sys.version_info[0:2])')
+        if [ "$JEDI_TEST_ENVIRONMENT" != "$python_ver" ]; then
+          echo "Unexpected Python version for $JEDI_TEST_ENVIRONMENT: $python_ver"
+          set +ex
+          exit 2
+        fi
+      fi
+      set +ex
     - tox
 after_script:
-    - if [ $TOXENV == "cov" ]; then
-      pip install --quiet --use-mirrors coveralls;
-      coveralls;
+    - |
+      if [ $TOXENV == "cov" ]; then
+        pip install --quiet codecov coveralls
+        coverage xml
+        coverage report -m
+        coveralls
+        bash <(curl -s https://codecov.io/bash) -X gcov -X coveragepy -X search -X fix -X xcode -f coverage.xml
       fi
-

AUTHORS.txt

@@ -1,4 +1,4 @@
-Main Authors
+Main Authors
 ============
 
 David Halter (@davidhalter) <davidhalter88@gmail.com>
@@ -40,6 +40,18 @@ Guido van Rossum (@gvanrossum) <guido@python.org>
 Dmytro Sadovnychyi (@sadovnychyi) <jedi@dmit.ro>
 Cristi Burcă (@scribu)
 bstaint (@bstaint)
-
+Mathias Rav (@Mortal) <rav@cs.au.dk>
+Daniel Fiterman (@dfit99) <fitermandaniel2@gmail.com>
+Simon Ruggier (@sruggier)
+Élie Gouzien (@ElieGouzien)
+Robin Roth (@robinro)
+Malte Plath (@langsamer)
+Anton Zub (@zabulazza)
+Maksim Novikov (@m-novikov) <mnovikov.work@gmail.com>
+Tobias Rzepka (@TobiasRzepka)
+micbou (@micbou)
+Dima Gerasimov (@karlicoss) <karlicoss@gmail.com>
+Max Woerner Chase (@mwchase) <max.chase@gmail.com>
+Johannes Maria Frank (@jmfrank63) <jmfrank63@gmail.com>
 
 Note: (@user) means a github user name.

CHANGELOG.rst

@@ -3,6 +3,120 @@
 Changelog
 ---------
 
+0.15.1 (2019-08-13)
++++++++++++++++++++
+
+- Small bugfix and removal of a print statement
+
+0.15.0 (2019-08-11)
++++++++++++++++++++
+
+- Added file path completions, there's a **new ``Completion.type``** ``path``,
+  now. Example: ``'/ho`` -> ``'/home/``
+- ``*args``/``**kwargs`` resolving. If possible Jedi replaces the parameters
+  with the actual alternatives.
+- Better support for enums/dataclasses
+- When using Interpreter, properties are now executed, since a lot of people
+  have complained about this. Discussion in #1299, #1347.
+
+New APIs:
+
+- ``Definition.get_signatures() -> List[Signature]``. Signatures are similar to
+  ``CallSignature``. ``Definition.params`` is therefore deprecated.
+- ``Signature.to_string()`` to format call signatures.
+- ``Signature.params -> List[ParamDefinition]``, ParamDefinition has the
+  following additional attributes ``infer_default()``, ``infer_annotation()``,
+  ``to_string()``, and ``kind``.
+- ``Definition.execute() -> List[Definition]``, makes it possible to infer
+    return values of functions.
+
+
+0.14.1 (2019-07-13)
++++++++++++++++++++
+
+- CallSignature.index should now be working a lot better
+- A couple of smaller bugfixes
+
+0.14.0 (2019-06-20)
++++++++++++++++++++
+
+- Added ``goto_*(prefer_stubs=True)`` as well as ``goto_*(prefer_stubs=True)``
+- Stubs are used now for type inference
+- Typeshed is used for better type inference
+- Reworked Definition.full_name, should have more correct return values
+
+0.13.3 (2019-02-24)
++++++++++++++++++++
+
+- Fixed an issue with embedded Python, see https://github.com/davidhalter/jedi-vim/issues/870
+
+0.13.2 (2018-12-15)
++++++++++++++++++++
+
+- Fixed a bug that led to Jedi spawning a lot of subprocesses.
+
+0.13.1 (2018-10-02)
++++++++++++++++++++
+
+- Bugfixes, because tensorflow completions were still slow.
+
+0.13.0 (2018-10-02)
++++++++++++++++++++
+
+- A small release. Some bug fixes.
+- Remove Python 3.3 support. Python 3.3 support has been dropped by the Python
+  foundation.
+- Default environments are now using the same Python version as the Python
+  process. In 0.12.x, we used to load the latest Python version on the system.
+- Added ``include_builtins`` as a parameter to usages.
+- ``goto_assignments`` has a new ``follow_builtin_imports`` parameter that
+  changes the previous behavior slightly.
+
+0.12.1 (2018-06-30)
++++++++++++++++++++
+
+- This release forces you to upgrade parso. If you don't, nothing will work
+  anymore. Otherwise changes should be limited to bug fixes. Unfortunately Jedi
+  still uses a few internals of parso that make it hard to keep compatibility
+  over multiple releases. Parso >=0.3.0 is going to be needed.
+
+0.12.0 (2018-04-15)
++++++++++++++++++++
+
+- Virtualenv/Environment support
+- F-String Completion/Goto Support
+- Cannot crash with segfaults anymore
+- Cleaned up import logic
+- Understand async/await and autocomplete it (including async generators)
+- Better namespace completions
+- Passing tests for Windows (including CI for Windows)
+- Remove Python 2.6 support
+
+0.11.1 (2017-12-14)
++++++++++++++++++++
+
+- Parso update - the caching layer was broken
+- Better usages - a lot of internal code was ripped out and improved.
+
+0.11.0 (2017-09-20)
++++++++++++++++++++
+
+- Split Jedi's parser into a separate project called ``parso``.
+- Avoiding side effects in REPL completion.
+- Numpy docstring support should be much better.
+- Moved the `settings.*recursion*` away, they are no longer usable.
+
+0.10.2 (2017-04-05)
++++++++++++++++++++
+
+- Python Packaging sucks. Some files were not included in 0.10.1.
+
+0.10.1 (2017-04-05)
++++++++++++++++++++
+
+- Fixed a few very annoying bugs.
+- Prepared the parser to be factored out of Jedi.
+
 0.10.0 (2017-02-03)
 +++++++++++++++++++
 

CONTRIBUTING.md

@@ -1,28 +1,8 @@
-Pull Requests are great (on the **dev** branch)! Readme/Documentation changes
-are ok in the master branch.
+Pull Requests are great.
 
  1. Fork the Repo on github.
  2. If you are adding functionality or fixing a bug, please add a test!
  3. Add your name to AUTHORS.txt
- 4. Push to your fork and submit a **pull request to the dev branch**.
+ 4. Push to your fork and submit a pull request.
 
-My **master** branch is a 100% stable (should be). I only push to it after I am
-certain that things are working out. Many people are using Jedi directly from
-the github master branch.
-
-**Try to use the PEP8 style guide.**
-
-
-Changing Issues to Pull Requests (Github)
------------------------------------------
-
-If you have have previously filed a GitHub issue and want to contribute code
-that addresses that issue, we prefer it if you use
-[hub](https://github.com/github/hub) to convert your existing issue to a pull
-request. To do that, first push the changes to a separate branch in your fork
-and then issue the following command:
-
-    hub pull-request -b davidhalter:dev -i <issue-number> -h <your-github-username>:<your-branch-name>
-
-It's no strict requirement though, if you don't have hub installed or prefer to
-use the web interface, then feel free to post a traditional pull request.
+**Try to use the PEP8 style guide** (and it's ok to have a line length of 100 characters).

LICENSE.txt

@@ -1,14 +1,5 @@
 All contributions towards Jedi are MIT licensed.
 
-Some Python files have been taken from the standard library and are therefore
-PSF licensed. Modifications on these files are dual licensed (both MIT and
-PSF). These files are:
-
-- jedi/parser/pgen2
-- jedi/parser/tokenize.py
-- jedi/parser/token.py
-- test/test_parser/test_pgen2.py
-
 -------------------------------------------------------------------------------
 The MIT License (MIT)
 
@@ -31,52 +22,3 @@ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 THE SOFTWARE.
--------------------------------------------------------------------------------
-
-PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2
---------------------------------------------
-
-1. This LICENSE AGREEMENT is between the Python Software Foundation
-("PSF"), and the Individual or Organization ("Licensee") accessing and
-otherwise using this software ("Python") in source or binary form and
-its associated documentation.
-
-2. Subject to the terms and conditions of this License Agreement, PSF hereby
-grants Licensee a nonexclusive, royalty-free, world-wide license to reproduce,
-analyze, test, perform and/or display publicly, prepare derivative works,
-distribute, and otherwise use Python alone or in any derivative version,
-provided, however, that PSF's License Agreement and PSF's notice of copyright,
-i.e., "Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010,
-2011, 2012, 2013, 2014, 2015 Python Software Foundation; All Rights Reserved"
-are retained in Python alone or in any derivative version prepared by Licensee.
-
-3. In the event Licensee prepares a derivative work that is based on
-or incorporates Python or any part thereof, and wants to make
-the derivative work available to others as provided herein, then
-Licensee hereby agrees to include in any such work a brief summary of
-the changes made to Python.
-
-4. PSF is making Python available to Licensee on an "AS IS"
-basis.  PSF MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR
-IMPLIED.  BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND
-DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS
-FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON WILL NOT
-INFRINGE ANY THIRD PARTY RIGHTS.
-
-5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON
-FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS
-A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON,
-OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF.
-
-6. This License Agreement will automatically terminate upon a material
-breach of its terms and conditions.
-
-7. Nothing in this License Agreement shall be deemed to create any
-relationship of agency, partnership, or joint venture between PSF and
-Licensee.  This License Agreement does not grant permission to use PSF
-trademarks or trade name in a trademark sense to endorse or promote
-products or services of Licensee, or any third party.
-
-8. By copying, installing or otherwise using Python, Licensee
-agrees to be bound by the terms and conditions of this License
-Agreement.

MANIFEST.in

@@ -7,8 +7,11 @@ include sith.py
 include conftest.py
 include pytest.ini
 include tox.ini
-include jedi/evaluate/compiled/fake/*.pym
-include jedi/parser/grammar*.txt
+include requirements.txt
+include jedi/parser/python/grammar*.txt
+recursive-include jedi/third_party *.pyi
+include jedi/third_party/typeshed/LICENSE
+include jedi/third_party/typeshed/README
 recursive-include test *
 recursive-include docs *
 recursive-exclude * *.pyc

README.rst

@@ -2,49 +2,59 @@
 Jedi - an awesome autocompletion/static analysis library for Python
 ###################################################################
 
-.. image:: https://secure.travis-ci.org/davidhalter/jedi.png?branch=master
-    :target: http://travis-ci.org/davidhalter/jedi
-    :alt: Travis-CI build status
+.. image:: https://img.shields.io/pypi/v/jedi.svg?style=flat
+    :target: https://pypi.python.org/pypi/jedi
+    :alt: PyPI version
 
-.. image:: https://coveralls.io/repos/davidhalter/jedi/badge.png?branch=master
+.. image:: https://img.shields.io/pypi/pyversions/jedi.svg
+    :target: https://pypi.python.org/pypi/jedi
+    :alt: Supported Python versions
+
+.. image:: https://travis-ci.org/davidhalter/jedi.svg?branch=master
+    :target: https://travis-ci.org/davidhalter/jedi
+    :alt: Linux Tests
+
+.. image:: https://ci.appveyor.com/api/projects/status/mgva3bbawyma1new/branch/master?svg=true
+    :target: https://ci.appveyor.com/project/davidhalter/jedi/branch/master
+    :alt: Windows Tests
+
+.. image:: https://coveralls.io/repos/davidhalter/jedi/badge.svg?branch=master
     :target: https://coveralls.io/r/davidhalter/jedi
-    :alt: Coverage Status
+    :alt: Coverage status
 
 
-*If you have specific questions, please add an issue or ask on* `stackoverflow
-<https://stackoverflow.com>`_ *with the label* ``python-jedi``.
+*If you have specific questions, please add an issue or ask on* `Stack Overflow
+<https://stackoverflow.com/questions/tagged/python-jedi>`_ *with the label* ``python-jedi``.
 
 
-Jedi is a static analysis tool for Python that can be used in IDEs/editors. Its
-historic focus is autocompletion, but does static analysis for now as well.
-Jedi is fast and is very well tested. It understands Python on a deeper level
-than all other static analysis frameworks for Python.
+Jedi is a static analysis tool for Python that can be used in IDEs/editors.
+Jedi has a focus on autocompletion and goto functionality. Jedi is fast and is
+very well tested. It understands Python and stubs on a deep level.
 
-Jedi has support for two different goto functions. It's possible to search for
-related names and to list all names in a Python file and infer them. Jedi
-understands docstrings and you can use Jedi autocompletion in your REPL as
-well.
+Jedi has support for different goto functions. It's possible to search for
+usages and list names in a Python file to get information about them.
 
 Jedi uses a very simple API to connect with IDE's. There's a reference
 implementation as a `VIM-Plugin <https://github.com/davidhalter/jedi-vim>`_,
 which uses Jedi's autocompletion.  We encourage you to use Jedi in your IDEs.
-It's really easy.
+Autocompletion in your REPL is also possible, IPython uses it natively and for
+the CPython REPL you have to install it.
 
 Jedi can currently be used with the following editors/projects:
 
-- Vim (jedi-vim_, YouCompleteMe_, deoplete-jedi_)
+- Vim (jedi-vim_, YouCompleteMe_, deoplete-jedi_, completor.vim_)
 - Emacs (Jedi.el_, company-mode_, elpy_, anaconda-mode_, ycmd_)
 - Sublime Text (SublimeJEDI_ [ST2 + ST3], anaconda_ [only ST3])
 - TextMate_ (Not sure if it's actually working)
 - Kate_ version 4.13+ supports it natively, you have to enable it, though. [`proof
   <https://projects.kde.org/projects/kde/applications/kate/repository/show?rev=KDE%2F4.13>`_]
-- Atom_ (autocomplete-python_)
-- SourceLair_
+- Atom_ (autocomplete-python-jedi_)
 - `GNOME Builder`_ (with support for GObject Introspection)
-- `Visual Studio Code`_ (via `Python Extension <https://marketplace.visualstudio.com/items?itemName=donjayamanne.python>`_)
+- `Visual Studio Code`_ (via `Python Extension <https://marketplace.visualstudio.com/items?itemName=ms-python.python>`_)
 - Gedit (gedi_)
 - wdb_ - Web Debugger
 - `Eric IDE`_ (Available as a plugin)
+- `IPython 6.0.0+ <https://ipython.readthedocs.io/en/stable/whatsnew/version6.html>`_
 
 and many more!
 
@@ -71,7 +81,11 @@ Get the latest version from `github <https://github.com/davidhalter/jedi>`_
 Docs are available at `https://jedi.readthedocs.org/en/latest/
 <https://jedi.readthedocs.org/en/latest/>`_. Pull requests with documentation
 enhancements and/or fixes are awesome and most welcome. Jedi uses `semantic
-versioning <http://semver.org/>`_.
+versioning <https://semver.org/>`_.
+
+If you want to stay up-to-date (News / RFCs), please subscribe to this `github
+thread <https://github.com/davidhalter/jedi/issues/1063>`_.:
+
 
 
 Installation
@@ -95,8 +109,10 @@ understands, see: `Features
 <https://jedi.readthedocs.org/en/latest/docs/features.html>`_. A list of
 caveats can be found on the same page.
 
-You can run Jedi on cPython 2.6, 2.7, 3.3, 3.4 or 3.5 but it should also
-understand/parse code older than those versions.
+You can run Jedi on CPython 2.7 or 3.4+ but it should also
+understand/parse code older than those versions. Additionally you should be able
+to use `Virtualenvs <https://jedi.readthedocs.org/en/latest/docs/api.html#environments>`_
+very well.
 
 Tips on how to use Jedi efficiently can be found `here
 <https://jedi.readthedocs.org/en/latest/docs/features.html#recipes>`_.
@@ -104,7 +120,7 @@ Tips on how to use Jedi efficiently can be found `here
 API
 ---
 
-You can find the documentation for the `API here <https://jedi.readthedocs.org/en/latest/docs/plugin-api.html>`_.
+You can find the documentation for the `API here <https://jedi.readthedocs.org/en/latest/docs/api.html>`_.
 
 
 Autocompletion / Goto / Pydoc
@@ -122,22 +138,20 @@ The returned objects are very powerful and really all you might need.
 Autocompletion in your REPL (IPython, etc.)
 -------------------------------------------
 
+Starting with IPython `6.0.0` Jedi is a dependency of IPython. Autocompletion
+in IPython is therefore possible without additional configuration.
+
 It's possible to have Jedi autocompletion in REPL modes - `example video <https://vimeo.com/122332037>`_.
-This means that IPython and others are `supported
+This means that in Python you can enable tab completion in a `REPL
 <https://jedi.readthedocs.org/en/latest/docs/usage.html#tab-completion-in-the-python-shell>`_.
 
 
-Static Analysis / Linter
+Static Analysis
 ------------------------
 
 To do all forms of static analysis, please try to use ``jedi.names``. It will
 return a list of names that you can use to infer types and so on.
 
-Linting is another thing that is going to be part of Jedi. For now you can try
-an alpha version ``python -m jedi linter``. The API might change though and
-it's still buggy. It's Jedi's goal to be smarter than classic linter and
-understand ``AttributeError`` and other code issues.
-
 
 Refactoring
 -----------
@@ -174,7 +188,7 @@ Tests are also run automatically on `Travis CI
 <https://travis-ci.org/davidhalter/jedi/>`_.
 
 For more detailed information visit the `testing documentation
-<https://jedi.readthedocs.org/en/latest/docs/testing.html>`_
+<https://jedi.readthedocs.org/en/latest/docs/testing.html>`_.
 
 
 Acknowledgements
@@ -189,8 +203,9 @@ Acknowledgements
 
 
 .. _jedi-vim: https://github.com/davidhalter/jedi-vim
-.. _youcompleteme: http://valloric.github.io/YouCompleteMe/
+.. _youcompleteme: https://github.com/ycm-core/YouCompleteMe
 .. _deoplete-jedi: https://github.com/zchee/deoplete-jedi
+.. _completor.vim: https://github.com/maralla/completor.vim
 .. _Jedi.el: https://github.com/tkf/emacs-jedi
 .. _company-mode: https://github.com/syohex/emacs-company-jedi
 .. _elpy: https://github.com/jorgenschaefer/elpy
@@ -200,11 +215,10 @@ Acknowledgements
 .. _anaconda: https://github.com/DamnWidget/anaconda
 .. _wdb: https://github.com/Kozea/wdb
 .. _TextMate: https://github.com/lawrenceakka/python-jedi.tmbundle
-.. _Kate: http://kate-editor.org
+.. _Kate: https://kate-editor.org
 .. _Atom: https://atom.io/
-.. _autocomplete-python: https://atom.io/packages/autocomplete-python
-.. _SourceLair: https://www.sourcelair.com
+.. _autocomplete-python-jedi: https://atom.io/packages/autocomplete-python-jedi
 .. _GNOME Builder: https://wiki.gnome.org/Apps/Builder
 .. _Visual Studio Code: https://code.visualstudio.com/
 .. _gedi: https://github.com/isamert/gedi
-.. _Eric IDE: http://eric-ide.python-projects.org
+.. _Eric IDE: https://eric-ide.python-projects.org

appveyor.yml

@@ -0,0 +1,87 @@
+environment:
+  matrix:
+  - TOXENV: py27
+    PYTHON_PATH: C:\Python27
+    JEDI_TEST_ENVIRONMENT: 27
+  - TOXENV: py27
+    PYTHON_PATH: C:\Python27
+    JEDI_TEST_ENVIRONMENT: 34
+  - TOXENV: py27
+    PYTHON_PATH: C:\Python27
+    JEDI_TEST_ENVIRONMENT: 35
+  - TOXENV: py27
+    PYTHON_PATH: C:\Python27
+    JEDI_TEST_ENVIRONMENT: 36
+  - TOXENV: py27
+    PYTHON_PATH: C:\Python27
+    JEDI_TEST_ENVIRONMENT: 37
+
+  - TOXENV: py34
+    PYTHON_PATH: C:\Python34
+    JEDI_TEST_ENVIRONMENT: 27
+  - TOXENV: py34
+    PYTHON_PATH: C:\Python34
+    JEDI_TEST_ENVIRONMENT: 34
+  - TOXENV: py34
+    PYTHON_PATH: C:\Python34
+    JEDI_TEST_ENVIRONMENT: 35
+  - TOXENV: py34
+    PYTHON_PATH: C:\Python34
+    JEDI_TEST_ENVIRONMENT: 36
+  - TOXENV: py34
+    PYTHON_PATH: C:\Python34
+    JEDI_TEST_ENVIRONMENT: 37
+
+  - TOXENV: py35
+    PYTHON_PATH: C:\Python35
+    JEDI_TEST_ENVIRONMENT: 27
+  - TOXENV: py35
+    PYTHON_PATH: C:\Python35
+    JEDI_TEST_ENVIRONMENT: 34
+  - TOXENV: py35
+    PYTHON_PATH: C:\Python35
+    JEDI_TEST_ENVIRONMENT: 35
+  - TOXENV: py35
+    PYTHON_PATH: C:\Python35
+    JEDI_TEST_ENVIRONMENT: 36
+  - TOXENV: py35
+    PYTHON_PATH: C:\Python35
+    JEDI_TEST_ENVIRONMENT: 37
+
+  - TOXENV: py36
+    PYTHON_PATH: C:\Python36
+    JEDI_TEST_ENVIRONMENT: 27
+  - TOXENV: py36
+    PYTHON_PATH: C:\Python36
+    JEDI_TEST_ENVIRONMENT: 34
+  - TOXENV: py36
+    PYTHON_PATH: C:\Python36
+    JEDI_TEST_ENVIRONMENT: 35
+  - TOXENV: py36
+    PYTHON_PATH: C:\Python36
+    JEDI_TEST_ENVIRONMENT: 36
+  - TOXENV: py36
+    PYTHON_PATH: C:\Python36
+    JEDI_TEST_ENVIRONMENT: 37
+
+  - TOXENV: py37
+    PYTHON_PATH: C:\Python37
+    JEDI_TEST_ENVIRONMENT: 27
+  - TOXENV: py37
+    PYTHON_PATH: C:\Python37
+    JEDI_TEST_ENVIRONMENT: 34
+  - TOXENV: py37
+    PYTHON_PATH: C:\Python37
+    JEDI_TEST_ENVIRONMENT: 35
+  - TOXENV: py37
+    PYTHON_PATH: C:\Python37
+    JEDI_TEST_ENVIRONMENT: 36
+  - TOXENV: py37
+    PYTHON_PATH: C:\Python37
+    JEDI_TEST_ENVIRONMENT: 37
+install:
+  - git submodule update --init --recursive
+  - set PATH=%PYTHON_PATH%;%PYTHON_PATH%\Scripts;%PATH%
+  - pip install tox
+build_script:
+  - tox

conftest.py

@@ -1,18 +1,28 @@
 import tempfile
 import shutil
+import os
+from functools import partial
 
 import pytest
 
 import jedi
+from jedi.api.environment import get_system_environment, InterpreterEnvironment
+from jedi._compatibility import py_version
 
-collect_ignore = ["setup.py"]
+collect_ignore = [
+    'setup.py',
+    '__main__.py',
+    'jedi/evaluate/compiled/subprocess/__main__.py',
+    'build/',
+    'test/examples',
+]
 
 
 # The following hooks (pytest_configure, pytest_unconfigure) are used
 # to modify `jedi.settings.cache_directory` because `clean_jedi_cache`
 # has no effect during doctests.  Without these hooks, doctests uses
 # user's cache (e.g., ~/.cache/jedi/).  We should remove this
-# workaround once the problem is fixed in py.test.
+# workaround once the problem is fixed in pytest.
 #
 # See:
 # - https://github.com/davidhalter/jedi/pull/168
@@ -29,6 +39,12 @@ def pytest_addoption(parser):
     parser.addoption("--warning-is-error", action='store_true',
                      help="Warnings are treated as errors.")
 
+    parser.addoption("--env", action='store',
+                     help="Execute the tests in that environment (e.g. 35 for python3.5).")
+    parser.addoption("--interpreter-env", "-I", action='store_true',
+                     help="Don't use subprocesses to guarantee having safe "
+                          "code execution. Useful for debugging.")
+
 
 def pytest_configure(config):
     global jedi_cache_directory_orig, jedi_cache_directory_temp
@@ -70,3 +86,73 @@ def clean_jedi_cache(request):
     def restore():
         settings.cache_directory = old
         shutil.rmtree(tmp)
+
+
+@pytest.fixture(scope='session')
+def environment(request):
+    if request.config.option.interpreter_env:
+        return InterpreterEnvironment()
+
+    version = request.config.option.env
+    if version is None:
+        version = os.environ.get('JEDI_TEST_ENVIRONMENT', str(py_version))
+
+    return get_system_environment(version[0] + '.' + version[1:])
+
+
+@pytest.fixture(scope='session')
+def Script(environment):
+    return partial(jedi.Script, environment=environment)
+
+
+@pytest.fixture(scope='session')
+def names(environment):
+    return partial(jedi.names, environment=environment)
+
+
+@pytest.fixture(scope='session')
+def has_typing(environment):
+    if environment.version_info >= (3, 5, 0):
+        # This if is just needed to avoid that tests ever skip way more than
+        # they should for all Python versions.
+        return True
+
+    script = jedi.Script('import typing', environment=environment)
+    return bool(script.goto_definitions())
+
+
+@pytest.fixture(scope='session')
+def jedi_path():
+    return os.path.dirname(__file__)
+
+
+@pytest.fixture()
+def skip_python2(environment):
+    if environment.version_info.major == 2:
+        # This if is just needed to avoid that tests ever skip way more than
+        # they should for all Python versions.
+        pytest.skip()
+
+
+@pytest.fixture()
+def skip_pre_python38(environment):
+    if environment.version_info < (3, 8):
+        # This if is just needed to avoid that tests ever skip way more than
+        # they should for all Python versions.
+        pytest.skip()
+
+
+@pytest.fixture()
+def skip_pre_python37(environment):
+    if environment.version_info < (3, 7):
+        # This if is just needed to avoid that tests ever skip way more than
+        # they should for all Python versions.
+        pytest.skip()
+
+
+@pytest.fixture()
+def skip_pre_python35(environment):
+    if environment.version_info < (3, 5):
+        # This if is just needed to avoid that tests ever skip way more than
+        # they should for all Python versions.
+        pytest.skip()

deploy-master.sh

@@ -0,0 +1,53 @@
+#!/usr/bin/env bash
+# The script creates a separate folder in build/ and creates tags there, pushes
+# them and then uploads the package to PyPI.
+
+set -eu -o pipefail
+
+BASE_DIR=$(dirname $(readlink -f "$0"))
+cd $BASE_DIR
+
+git fetch --tags
+
+PROJECT_NAME=jedi
+BRANCH=master
+BUILD_FOLDER=build
+
+[ -d $BUILD_FOLDER ] || mkdir $BUILD_FOLDER
+# Remove the previous deployment first.
+# Checkout the right branch
+cd $BUILD_FOLDER
+rm -rf $PROJECT_NAME
+git clone .. $PROJECT_NAME
+cd $PROJECT_NAME
+git checkout $BRANCH
+git submodule update --init
+
+# Test first.
+tox
+
+# Create tag
+tag=v$(python -c "import $PROJECT_NAME; print($PROJECT_NAME.__version__)")
+
+master_ref=$(git show-ref -s heads/$BRANCH)
+tag_ref=$(git show-ref -s $tag || true)
+if [[ $tag_ref ]]; then
+    if [[ $tag_ref != $master_ref ]]; then
+        echo 'Cannot tag something that has already been tagged with another commit.'
+        exit 1
+    fi
+else
+    git tag -a $tag
+    git push --tags
+fi
+
+# Package and upload to PyPI
+#rm -rf dist/ - Not needed anymore, because the folder is never reused.
+echo `pwd`
+python setup.py sdist bdist_wheel
+# Maybe do a pip install twine before.
+twine upload dist/*
+
+cd $BASE_DIR
+# The tags have been pushed to this repo. Push the tags to github, now.
+git push --tags

docs/_templates/ghbuttons.html

@@ -1,4 +1,4 @@
 <h3>Github</h3>
-<iframe src="http://ghbtns.com/github-btn.html?user=davidhalter&repo=jedi&type=watch&count=true&size=large"
+<iframe src="https://ghbtns.com/github-btn.html?user=davidhalter&repo=jedi&type=watch&count=true&size=large"
         frameborder="0" scrolling="0" width="170" height="30" allowtransparency="true"></iframe>
 <br><br>

docs/_themes/flask/layout.html

@@ -19,7 +19,6 @@
 {% endblock %}
 {%- block footer %}
   <div class="footer">
-    &copy; Copyright {{ copyright }}.
     Created using <a href="http://sphinx.pocoo.org/">Sphinx</a>.
   </div>
   {% if pagename == 'index' %}

docs/conf.py

@@ -45,7 +45,7 @@ master_doc = 'index'
 
 # General information about the project.
 project = u'Jedi'
-copyright = u'2012 - {today.year}, Jedi contributors'.format(today=datetime.date.today())
+copyright = u'jedi contributors'
 
 import jedi
 from jedi.utils import version_info
@@ -274,7 +274,7 @@ autodoc_default_flags = []
 # -- Options for intersphinx module --------------------------------------------
 
 intersphinx_mapping = {
-    'http://docs.python.org/': None,
+    'https://docs.python.org/': None,
 }
 
 

docs/docs/api-classes.rst

@@ -1,6 +1,6 @@
 .. include:: ../global.rst
 
-.. _plugin-api-classes:
+.. _api-classes:
 
 API Return Classes
 ------------------

docs/docs/api.rst

@@ -1,7 +1,7 @@
 .. include:: ../global.rst
 
-The Plugin API
-==============
+API Overview
+============
 
 .. currentmodule:: jedi
 
@@ -11,7 +11,7 @@ editors/IDE autocompletion
 If you want to use |jedi|, you first need to ``import jedi``.  You then have
 direct access to the :class:`.Script`. You can then call the functions
 documented here. These functions return :ref:`API classes
-<plugin-api-classes>`.
+<api-classes>`.
 
 
 Deprecations
@@ -24,16 +24,48 @@ The deprecation process is as follows:
    the deprecated functionality.
 
 
-API documentation
+API Documentation
 -----------------
 
-API Interface
-~~~~~~~~~~~~~
+The API consists of a few different parts:
 
-.. automodule:: jedi.api
+- The main starting points for completions/goto: :class:`.Script` and :class:`.Interpreter`
+- Helpful functions: :func:`.names`, :func:`.preload_module` and
+  :func:`.set_debug_function`
+- :ref:`API Result Classes <api-classes>`
+- :ref:`Python Versions/Virtualenv Support <environments>` with functions like
+  :func:`.find_system_environments` and :func:`.find_virtualenvs`
+
+.. _api:
+
+Static Analysis Interface
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. automodule:: jedi
+
+.. autoclass:: jedi.Script
     :members:
-    :undoc-members:
+.. autoclass:: jedi.Interpreter
+    :members:
+.. autofunction:: jedi.names
+.. autofunction:: jedi.preload_module
+.. autofunction:: jedi.set_debug_function
 
+.. _environments:
+
+Environments
+~~~~~~~~~~~~
+
+.. automodule:: jedi.api.environment
+
+.. autofunction:: jedi.find_system_environments
+.. autofunction:: jedi.find_virtualenvs
+.. autofunction:: jedi.get_system_environment
+.. autofunction:: jedi.create_environment
+.. autofunction:: jedi.get_default_environment
+.. autoexception:: jedi.InvalidPythonEnvironment
+.. autoclass:: jedi.api.environment.Environment
+    :members:
 
 Examples
 --------

docs/docs/development.rst

@@ -7,14 +7,16 @@ Jedi Development
 
 .. note:: This documentation is for Jedi developers who want to improve Jedi
     itself, but have no idea how Jedi works. If you want to use Jedi for 
-    your IDE, look at the `plugin api <plugin-api.html>`_.
+    your IDE, look at the `plugin api <api.html>`_.
+    It is also important to note that it's a pretty old version and some things
+    might not apply anymore.
 
 
 Introduction
 ------------
 
 This page tries to address the fundamental demand for documentation of the
-|jedi| interals. Understanding a dynamic language is a complex task. Especially
+|jedi| internals. Understanding a dynamic language is a complex task. Especially
 because type inference in Python can be a very recursive task. Therefore |jedi|
 couldn't get rid of complexity. I know that **simple is better than complex**,
 but unfortunately it sometimes requires complex solutions to understand complex
@@ -54,31 +56,15 @@ because that's where all the magic happens. I need to introduce the :ref:`parser
 
 .. _parser:
 
-Parser (parser/__init__.py)
+Parser
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-.. automodule:: jedi.parser
+Jedi used to have it's internal parser, however this is now a separate project
+and is called `parso <http://parso.readthedocs.io>`_.
 
-Parser Tree (parser/tree.py)
-++++++++++++++++++++++++++++++++++++++++++++++++
-
-.. automodule:: jedi.parser.tree
-
-Class inheritance diagram:
-
-.. inheritance-diagram::
-   Module
-   Class
-   Function
-   Lambda
-   Flow
-   ForStmt
-   Import
-   ExprStmt
-   Param
-   Name
-   CompFor
-   :parts: 1
+The parser creates a syntax tree that |jedi| analyses and tries to understand.
+The grammar that this parsers uses is very similar to the official Python
+`grammar files <https://docs.python.org/3/reference/grammar.html>`_.
 
 .. _evaluate:
 
@@ -87,16 +73,16 @@ Evaluation of python code (evaluate/__init__.py)
 
 .. automodule:: jedi.evaluate
 
-Evaluation Representation (evaluate/representation.py)
+Evaluation Contexts (evaluate/base_context.py)
 ++++++++++++++++++++++++++++++++++++++++++++++++++++++
 
-.. automodule:: jedi.evaluate.representation
+.. automodule:: jedi.evaluate.base_context
 
 .. inheritance-diagram::
-   jedi.evaluate.instance.TreeInstance
-   jedi.evaluate.representation.ClassContext
-   jedi.evaluate.representation.FunctionContext
-   jedi.evaluate.representation.FunctionExecutionContext
+   jedi.evaluate.context.instance.TreeInstance
+   jedi.evaluate.context.klass.ClassContext
+   jedi.evaluate.context.function.FunctionContext
+   jedi.evaluate.context.function.FunctionExecutionContext
    :parts: 1
 
 
@@ -110,11 +96,11 @@ Name resolution (evaluate/finder.py)
 
 .. _dev-api:
 
-API (api.py and api_classes.py)
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+API (api/__init__.py and api/classes.py)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 The API has been designed to be as easy to use as possible. The API
-documentation can be found `here <plugin-api.html>`_. The API itself contains
+documentation can be found `here <api.html>`_. The API itself contains
 little code that needs to be mentioned here. Generally I'm trying to be
 conservative with the API.  I'd rather not add new API features if they are not
 necessary, because it's much harder to deprecate stuff than to add it later.
@@ -129,7 +115,6 @@ Core Extensions is a summary of the following topics:
 
 - :ref:`Iterables & Dynamic Arrays <iterables>`
 - :ref:`Dynamic Parameters <dynamic>`
-- :ref:`Diff Parser <diff-parser>`
 - :ref:`Docstrings <docstrings>`
 - :ref:`Refactoring <refactoring>`
 
@@ -139,13 +124,13 @@ without some features.
 
 .. _iterables:
 
-Iterables & Dynamic Arrays (evaluate/iterable.py)
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Iterables & Dynamic Arrays (evaluate/context/iterable.py)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 To understand Python on a deeper level, |jedi| needs to understand some of the
 dynamic features of Python like lists that are filled after creation:
 
-.. automodule:: jedi.evaluate.iterable
+.. automodule:: jedi.evaluate.context.iterable
 
 
 .. _dynamic:
@@ -156,13 +141,6 @@ Parameter completion (evaluate/dynamic.py)
 .. automodule:: jedi.evaluate.dynamic
 
 
-.. _diff-parser:
-
-Diff Parser (parser/diff.py)
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. automodule:: jedi.parser.diff
-
 .. _docstrings:
 
 Docstrings (evaluate/docstrings.py)

docs/docs/features.rst

@@ -8,9 +8,6 @@ Jedi obviously supports autocompletion. It's also possible to get it working in
 
 Static analysis is also possible by using the command ``jedi.names``.
 
-The Jedi Linter is currently in an alpha version and can be tested by calling
-``python -m jedi linter``.
-
 Jedi would in theory support refactoring, but we have never publicized it,
 because it's not production ready. If you're interested in helping out here,
 let me know. With the latest parser changes, it should be very easy to actually
@@ -20,12 +17,13 @@ make it work.
 General Features
 ----------------
 
-- python 2.6+ and 3.3+ support
-- ignores syntax errors and wrong indentation
-- can deal with complex module / function / class structures
-- virtualenv support
-- can infer function arguments from sphinx, epydoc and basic numpydoc docstrings,
+- Python 2.7 and 3.4+ support
+- Ignores syntax errors and wrong indentation
+- Can deal with complex module / function / class structures
+- Great Virtualenv support
+- Can infer function arguments from sphinx, epydoc and basic numpydoc docstrings,
   and PEP0484-style type hints (:ref:`type hinting <type-hinting>`)
+- Stub files
 
 
 Supported Python Features
@@ -47,24 +45,22 @@ Supported Python Features
 - (nested) list comprehensions / ternary expressions
 - relative imports
 - ``getattr()`` / ``__getattr__`` / ``__getattribute__``
-- function annotations (py3k feature, are ignored right now, but being parsed.
-  I don't know what to do with them.)
+- function annotations
 - class decorators (py3k feature, are being ignored too, until I find a use
   case, that doesn't work with |jedi|)
 - simple/usual ``sys.path`` modifications
 - ``isinstance`` checks for if/while/assert
-- namespace packages (includes ``pkgutil`` and ``pkg_resources`` namespaces)
+- namespace packages (includes ``pkgutil``, ``pkg_resources`` and PEP420 namespaces)
 - Django / Flask / Buildout support
 
 
-Unsupported Features
---------------------
+Not Supported
+-------------
 
 Not yet implemented:
 
 - manipulations of instances outside the instance variables without using
   methods
-- implicit namespace packages (Python 3.3+, `PEP 420 <https://www.python.org/dev/peps/pep-0420/>`_)
 
 Will probably never be implemented:
 
@@ -77,21 +73,6 @@ Will probably never be implemented:
 Caveats
 -------
 
-**Malformed Syntax**
-
-Syntax errors and other strange stuff may lead to undefined behaviour of the
-completion. |jedi| is **NOT** a Python compiler, that tries to correct you. It
-is a tool that wants to help you. But **YOU** have to know Python, not |jedi|.
-
-**Legacy Python 2 Features**
-
-This framework should work for both Python 2/3. However, some things were just
-not as *pythonic* in Python 2 as things should be. To keep things simple, some
-older Python 2 features have been left out:
-
-- Classes: Always Python 3 like, therefore all classes inherit from ``object``.
-- Generators: No ``next()`` method. The ``__next__()`` method is used instead.
-
 **Slow Performance**
 
 Importing ``numpy`` can be quite slow sometimes, as well as loading the
@@ -103,13 +84,13 @@ etc.
 **Security**
 
 Security is an important issue for |jedi|. Therefore no Python code is
-executed.  As long as you write pure python, everything is evaluated
+executed.  As long as you write pure Python, everything is evaluated
 statically. But: If you use builtin modules (``c_builtin``) there is no other
 option than to execute those modules. However: Execute isn't that critical (as
 e.g. in pythoncomplete, which used to execute *every* import!), because it
 means one import and no more.  So basically the only dangerous thing is using
 the import itself. If your ``c_builtin`` uses some strange initializations, it
-might be dangerous. But if it does you're screwed anyways, because eventualy
+might be dangerous. But if it does you're screwed anyways, because eventually
 you're going to execute your code, which executes the import.
 
 
@@ -132,8 +113,7 @@ one of the following docstring/annotation syntax styles:
 
 https://www.python.org/dev/peps/pep-0484/
 
-function annotations (python 3 only; python 2 function annotations with
-comments in planned but not yet implemented)
+function annotations
 
 ::
 
@@ -144,7 +124,7 @@ comments in planned but not yet implemented)
         node.| # complete here
 
 
-assignment, for-loop and with-statement type hints (all python versions).
+assignment, for-loop and with-statement type hints (all Python versions).
 Note that the type hints must be on the same line as the statement
 
 ::
@@ -157,22 +137,15 @@ Note that the type hints must be on the same line as the statement
         print(f + 3)
 
 Most of the features in PEP-0484 are supported including the typing module
-(for python < 3.5 you have to do ``pip install typing`` to use these),
+(for Python < 3.5 you have to do ``pip install typing`` to use these),
 and forward references.
 
-Things that are missing (and this is not an exhaustive list; some of these
-are planned, others might be hard to implement and provide little worth):
+You can also use stub files.
 
-- annotating functions with comments: https://www.python.org/dev/peps/pep-0484/#suggested-syntax-for-python-2-7-and-straddling-code
-- understanding ``typing.cast()``
-- stub files: https://www.python.org/dev/peps/pep-0484/#stub-files
-- ``typing.Callable``
-- ``typing.TypeVar``
-- User defined generic types: https://www.python.org/dev/peps/pep-0484/#user-defined-generic-types
 
 **Sphinx style**
 
-http://sphinx-doc.org/domains.html#info-field-lists
+http://www.sphinx-doc.org/en/stable/domains.html#info-field-lists
 
 ::
 
@@ -237,7 +210,7 @@ A little history
 
 The Star Wars Jedi are awesome. My Jedi software tries to imitate a little bit
 of the precognition the Jedi have. There's even an awesome `scene
-<http://www.youtube.com/watch?v=5BDO3pyavOY>`_ of Monty Python Jedis :-).
+<https://youtu.be/yHRJLIf7wMU>`_ of Monty Python Jedis :-).
 
 But actually the name hasn't so much to do with Star Wars. It's part of my
 second name.

docs/docs/installation.rst

@@ -11,8 +11,16 @@ jedi-vim_ does by default), or you can install it systemwide.
     editor, refer to the corresponding documentation.
 
 
-The preferred way
------------------
+The normal way
+--------------
+
+Most people use Jedi with a :ref:`editor plugins<editor-plugins>`. Typically
+you install Jedi by installing an editor plugin. No necessary steps are needed.
+Just take a look at the instructions for the plugin.
+
+
+With pip
+--------
 
 On any system you can install |jedi| directly from the Python package index
 using pip::
@@ -49,7 +57,7 @@ Debian
 ~~~~~~
 
 Debian packages are available in the `unstable repository
-<http://packages.debian.org/search?keywords=python%20jedi>`__.
+<https://packages.debian.org/search?keywords=python%20jedi>`__.
 
 Others
 ~~~~~~
@@ -57,19 +65,15 @@ Others
 We are in the discussion of adding |jedi| to the Fedora repositories.
 
 
-Manual installation from a downloaded package
+Manual installation from GitHub
 ---------------------------------------------
 
-If you prefer not to use an automated package installer, you can `download
-<https://github.com/davidhalter/jedi/archive/master.zip>`__ a current copy of
-|jedi| and install it manually.
-
-To install it, navigate to the directory containing `setup.py` on your console
-and type::
+If you prefer not to use an automated package installer, you can clone the source from GitHub and install it manually. To install it, run these commands::
 
+    git clone --recurse-submodules https://github.com/davidhalter/jedi
+    cd jedi
     sudo python setup.py install
 
-
 Inclusion as a submodule
 ------------------------
 

docs/docs/static_analsysis.rst

@@ -1,106 +0,0 @@
-
-This file is the start of the documentation of how static analysis works.
-
-Below is a list of parser names that are used within nodes_to_execute.
-
------------- cared for:
-global_stmt
-exec_stmt  # no priority
-assert_stmt
-if_stmt
-while_stmt
-for_stmt
-try_stmt
-(except_clause)
-with_stmt
-(with_item)
-(with_var)
-print_stmt
-del_stmt
-return_stmt
-raise_stmt
-yield_expr
-file_input
-funcdef
-param
-old_lambdef
-lambdef
-import_name
-import_from
-(import_as_name)
-(dotted_as_name)
-(import_as_names)
-(dotted_as_names)
-(dotted_name)
-classdef
-comp_for
-(comp_if) ?
-decorator
-
------------ add basic
-test
-or_test
-and_test
-not_test
-expr
-xor_expr
-and_expr
-shift_expr
-arith_expr
-term
-factor
-power
-atom
-comparison
-expr_stmt
-testlist
-testlist1
-testlist_safe
-
------------ special care:
-# mostly depends on how we handle the other ones.
-testlist_star_expr  # should probably just work with expr_stmt
-star_expr
-exprlist  # just ignore? then names are just resolved. Strange anyway, bc expr is not really allowed in the list, typically.
-
------------ ignore:
-suite
-subscriptlist
-subscript
-simple_stmt
-?? sliceop     # can probably just be added.
-testlist_comp  # prob ignore and care about it with atom.
-dictorsetmaker
-trailer
-decorators
-decorated
-# always execute function arguments? -> no problem with stars.
-# Also arglist and argument are different in different grammars.
-arglist
-argument
-
-
------------ remove:
-tname      # only exists in current Jedi parser. REMOVE!
-tfpdef     # python 2: tuple assignment; python 3: annotation
-vfpdef     # reduced in python 3 and therefore not existing.
-tfplist    # not in 3
-vfplist    # not in 3
-
---------- not existing with parser reductions.
-small_stmt
-import_stmt
-flow_stmt
-compound_stmt
-stmt
-pass_stmt
-break_stmt
-continue_stmt
-comp_op
-augassign
-old_test
-typedargslist  # afaik becomes [param]
-varargslist    # dito
-vname
-comp_iter
-test_nocond

docs/docs/usage.rst

@@ -54,11 +54,7 @@ Visual Studio Code:
 
 Atom:
 
-- autocomplete-python_
-
-SourceLair:
-
-- SourceLair_
+- autocomplete-python-jedi_
 
 GNOME Builder:
 
@@ -82,9 +78,12 @@ and many more!
 
 .. _repl-completion:
 
-Tab completion in the Python Shell
+Tab Completion in the Python Shell
 ----------------------------------
 
+Starting with Ipython `6.0.0` Jedi is a dependency of IPython. Autocompletion
+in IPython is therefore possible without additional configuration.
+
 There are two different options how you can use Jedi autocompletion in
 your Python interpreter. One with your custom ``$HOME/.pythonrc.py`` file
 and one that uses ``PYTHONSTARTUP``.
@@ -100,7 +99,7 @@ Using a custom ``$HOME/.pythonrc.py``
 .. autofunction:: jedi.utils.setup_readline
 
 .. _jedi-vim: https://github.com/davidhalter/jedi-vim
-.. _youcompleteme: http://valloric.github.io/YouCompleteMe/
+.. _youcompleteme: https://valloric.github.io/YouCompleteMe/
 .. _deoplete-jedi: https://github.com/zchee/deoplete-jedi
 .. _Jedi.el: https://github.com/tkf/emacs-jedi
 .. _elpy: https://github.com/jorgenschaefer/elpy
@@ -110,10 +109,9 @@ Using a custom ``$HOME/.pythonrc.py``
 .. _SynJedi: http://uvviewsoft.com/synjedi/
 .. _wdb: https://github.com/Kozea/wdb
 .. _TextMate: https://github.com/lawrenceakka/python-jedi.tmbundle
-.. _kate: http://kate-editor.org/
-.. _autocomplete-python: https://atom.io/packages/autocomplete-python
-.. _SourceLair: https://www.sourcelair.com
+.. _kate: https://kate-editor.org/
+.. _autocomplete-python-jedi: https://atom.io/packages/autocomplete-python-jedi
 .. _GNOME Builder: https://wiki.gnome.org/Apps/Builder/
 .. _gedi: https://github.com/isamert/gedi
-.. _Eric IDE: http://eric-ide.python-projects.org
+.. _Eric IDE: https://eric-ide.python-projects.org
 .. _Python Extension: https://marketplace.visualstudio.com/items?itemName=donjayamanne.python

docs/index.rst

@@ -23,8 +23,8 @@ Docs
    docs/usage
    docs/installation
    docs/features
-   docs/plugin-api
-   docs/plugin-api-classes
+   docs/api
+   docs/api-classes
    docs/settings
    docs/development
    docs/testing
@@ -37,4 +37,4 @@ Resources
 
 - `Source Code on Github <https://github.com/davidhalter/jedi>`_
 - `Travis Testing <https://travis-ci.org/davidhalter/jedi>`_
-- `Python Package Index <http://pypi.python.org/pypi/jedi/>`_
+- `Python Package Index <https://pypi.python.org/pypi/jedi/>`_

jedi/__init__.py

@@ -1,43 +1,47 @@
 """
-Jedi is a static analysis tool for Python that can be used in IDEs/editors. Its
-historic focus is autocompletion, but does static analysis for now as well.
-Jedi is fast and is very well tested. It understands Python on a deeper level
-than all other static analysis frameworks for Python.
+Jedi is a static analysis tool for Python that can be used in IDEs/editors.
+Jedi has a focus on autocompletion and goto functionality. Jedi is fast and is
+very well tested. It understands Python and stubs on a deep level.
 
-Jedi has support for two different goto functions. It's possible to search for
-related names and to list all names in a Python file and infer them. Jedi
-understands docstrings and you can use Jedi autocompletion in your REPL as
-well.
+Jedi has support for different goto functions. It's possible to search for
+usages and list names in a Python file to get information about them.
 
 Jedi uses a very simple API to connect with IDE's. There's a reference
 implementation as a `VIM-Plugin <https://github.com/davidhalter/jedi-vim>`_,
 which uses Jedi's autocompletion.  We encourage you to use Jedi in your IDEs.
-It's really easy.
+Autocompletion in your REPL is also possible, IPython uses it natively and for
+the CPython REPL you have to install it.
 
-To give you a simple example how you can use the Jedi library, here is an
-example for the autocompletion feature:
+Here's a simple example of the autocompletion feature:
 
 >>> import jedi
 >>> source = '''
-... import datetime
-... datetime.da'''
->>> script = jedi.Script(source, 3, len('datetime.da'), 'example.py')
+... import json
+... json.lo'''
+>>> script = jedi.Script(source, 3, len('json.lo'), 'example.py')
 >>> script
-<Script: 'example.py'>
+<Script: 'example.py' ...>
 >>> completions = script.completions()
->>> completions                                         #doctest: +ELLIPSIS
-[<Completion: date>, <Completion: datetime>, ...]
+>>> completions
+[<Completion: load>, <Completion: loads>]
 >>> print(completions[0].complete)
-te
+ad
 >>> print(completions[0].name)
-date
+load
 
 As you see Jedi is pretty simple and allows you to concentrate on writing a
 good text editor, while still having very good IDE features for Python.
 """
 
-__version__ = '0.10.0'
+__version__ = '0.15.1'
 
-from jedi.api import Script, Interpreter, NotFoundError, set_debug_function
-from jedi.api import preload_module, defined_names, names
+from jedi.api import Script, Interpreter, set_debug_function, \
+    preload_module, names
 from jedi import settings
+from jedi.api.environment import find_virtualenvs, find_system_environments, \
+    get_default_environment, InvalidPythonEnvironment, create_environment, \
+    get_system_environment
+from jedi.api.exceptions import InternalError
+# Finally load the internal plugins. This is only internal.
+from jedi.plugins import registry
+del registry

jedi/_compatibility.py

@@ -1,24 +1,29 @@
 """
-To ensure compatibility from Python ``2.6`` - ``3.3``, a module has been
+To ensure compatibility from Python ``2.7`` - ``3.x``, a module has been
 created. Clearly there is huge need to use conforming syntax.
 """
+from __future__ import print_function
+import atexit
+import errno
+import functools
 import sys
-import imp
 import os
 import re
 import pkgutil
+import warnings
+import inspect
+import subprocess
+import weakref
 try:
     import importlib
 except ImportError:
     pass
+from zipimport import zipimporter
+
+from jedi.file_io import KnownContentFileIO, ZipFileIO
 
-# Cannot use sys.version.major and minor names, because in Python 2.6 it's not
-# a namedtuple.
 is_py3 = sys.version_info[0] >= 3
-is_py33 = is_py3 and sys.version_info[1] >= 3
-is_py34 = is_py3 and sys.version_info[1] >= 4
 is_py35 = is_py3 and sys.version_info[1] >= 5
-is_py26 = not is_py3 and sys.version_info[1] < 7
 py_version = int(str(sys.version_info[0]) + str(sys.version_info[1]))
 
 
@@ -34,58 +39,130 @@ class DummyFile(object):
         del self.loader
 
 
-def find_module_py33(string, path=None):
-    loader = importlib.machinery.PathFinder.find_module(string, path)
+def find_module_py34(string, path=None, full_name=None, is_global_search=True):
+    spec = None
+    loader = None
+
+    for finder in sys.meta_path:
+        if is_global_search and finder != importlib.machinery.PathFinder:
+            p = None
+        else:
+            p = path
+        try:
+            find_spec = finder.find_spec
+        except AttributeError:
+            # These are old-school clases that still have a different API, just
+            # ignore those.
+            continue
+
+        spec = find_spec(string, p)
+        if spec is not None:
+            loader = spec.loader
+            if loader is None and not spec.has_location:
+                # This is a namespace package.
+                full_name = string if not path else full_name
+                implicit_ns_info = ImplicitNSInfo(full_name, spec.submodule_search_locations._path)
+                return implicit_ns_info, True
+            break
+
+    return find_module_py33(string, path, loader)
+
+
+def find_module_py33(string, path=None, loader=None, full_name=None, is_global_search=True):
+    loader = loader or importlib.machinery.PathFinder.find_module(string, path)
 
     if loader is None and path is None:  # Fallback to find builtins
         try:
-            loader = importlib.find_loader(string)
+            with warnings.catch_warnings(record=True):
+                # Mute "DeprecationWarning: Use importlib.util.find_spec()
+                # instead." While we should replace that in the future, it's
+                # probably good to wait until we deprecate Python 3.3, since
+                # it was added in Python 3.4 and find_loader hasn't been
+                # removed in 3.6.
+                loader = importlib.find_loader(string)
         except ValueError as e:
             # See #491. Importlib might raise a ValueError, to avoid this, we
             # just raise an ImportError to fix the issue.
             raise ImportError("Originally  " + repr(e))
 
     if loader is None:
-        raise ImportError("Couldn't find a loader for {0}".format(string))
+        raise ImportError("Couldn't find a loader for {}".format(string))
 
+    return _from_loader(loader, string)
+
+
+def _from_loader(loader, string):
+    is_package = loader.is_package(string)
     try:
-        is_package = loader.is_package(string)
-        if is_package:
-            if hasattr(loader, 'path'):
-                module_path = os.path.dirname(loader.path)
-            else:
-                # At least zipimporter does not have path attribute
-                module_path = os.path.dirname(loader.get_filename(string))
-            if hasattr(loader, 'archive'):
-                module_file = DummyFile(loader, string)
-            else:
-                module_file = None
-        else:
-            module_path = loader.get_filename(string)
-            module_file = DummyFile(loader, string)
+        get_filename = loader.get_filename
     except AttributeError:
-        # ExtensionLoader has not attribute get_filename, instead it has a
-        # path attribute that we can use to retrieve the module path
-        try:
-            module_path = loader.path
-            module_file = DummyFile(loader, string)
-        except AttributeError:
-            module_path = string
-            module_file = None
-        finally:
-            is_package = False
+        return None, is_package
+    else:
+        module_path = cast_path(get_filename(string))
 
-    if hasattr(loader, 'archive'):
-        module_path = loader.archive
+    # To avoid unicode and read bytes, "overwrite" loader.get_source if
+    # possible.
+    f = type(loader).get_source
+    if is_py3 and f is not importlib.machinery.SourceFileLoader.get_source:
+        # Unfortunately we are reading unicode here, not bytes.
+        # It seems hard to get bytes, because the zip importer
+        # logic just unpacks the zip file and returns a file descriptor
+        # that we cannot as easily access. Therefore we just read it as
+        # a string in the cases where get_source was overwritten.
+        code = loader.get_source(string)
+    else:
+        code = _get_source(loader, string)
 
-    return module_file, module_path, is_package
+    if code is None:
+        return None, is_package
+    if isinstance(loader, zipimporter):
+        return ZipFileIO(module_path, code, cast_path(loader.archive)), is_package
+
+    return KnownContentFileIO(module_path, code), is_package
 
 
-def find_module_pre_py33(string, path=None):
+def _get_source(loader, fullname):
+    """
+    This method is here as a replacement for SourceLoader.get_source. That
+    method returns unicode, but we prefer bytes.
+    """
+    path = loader.get_filename(fullname)
+    try:
+        return loader.get_data(path)
+    except OSError:
+        raise ImportError('source not available through get_data()',
+                          name=fullname)
+
+
+def find_module_pre_py3(string, path=None, full_name=None, is_global_search=True):
+    # This import is here, because in other places it will raise a
+    # DeprecationWarning.
+    import imp
     try:
         module_file, module_path, description = imp.find_module(string, path)
         module_type = description[2]
-        return module_file, module_path, module_type is imp.PKG_DIRECTORY
+        is_package = module_type is imp.PKG_DIRECTORY
+        if is_package:
+            # In Python 2 directory package imports are returned as folder
+            # paths, not __init__.py paths.
+            p = os.path.join(module_path, '__init__.py')
+            try:
+                module_file = open(p)
+                module_path = p
+            except FileNotFoundError:
+                pass
+        elif module_type != imp.PY_SOURCE:
+            if module_file is not None:
+                module_file.close()
+            module_file = None
+
+        if module_file is None:
+            code = None
+            return None, is_package
+
+        with module_file:
+            code = module_file.read()
+        return KnownContentFileIO(cast_path(module_path), code), is_package
     except ImportError:
         pass
 
@@ -94,33 +171,13 @@ def find_module_pre_py33(string, path=None):
     for item in path:
         loader = pkgutil.get_importer(item)
         if loader:
-            try:
-                loader = loader.find_module(string)
-                if loader:
-                    is_package = loader.is_package(string)
-                    is_archive = hasattr(loader, 'archive')
-                    try:
-                        module_path = loader.get_filename(string)
-                    except AttributeError:
-                        # fallback for py26
-                        try:
-                            module_path = loader._get_filename(string)
-                        except AttributeError:
-                            continue
-                    if is_package:
-                        module_path = os.path.dirname(module_path)
-                    if is_archive:
-                        module_path = loader.archive
-                    file = None
-                    if not is_package or is_archive:
-                        file = DummyFile(loader, string)
-                    return (file, module_path, is_package)
-            except ImportError:
-                pass
-    raise ImportError("No module named {0}".format(string))
+            loader = loader.find_module(string)
+            if loader is not None:
+                return _from_loader(loader, string)
+    raise ImportError("No module named {}".format(string))
 
 
-find_module = find_module_py33 if is_py33 else find_module_pre_py33
+find_module = find_module_py34 if is_py3 else find_module_pre_py3
 find_module.__doc__ = """
 Provides information about a module.
 
@@ -132,28 +189,87 @@ if the module is contained in a package.
 """
 
 
+def _iter_modules(paths, prefix=''):
+    # Copy of pkgutil.iter_modules adapted to work with namespaces
+
+    for path in paths:
+        importer = pkgutil.get_importer(path)
+
+        if not isinstance(importer, importlib.machinery.FileFinder):
+            # We're only modifying the case for FileFinder. All the other cases
+            # still need to be checked (like zip-importing). Do this by just
+            # calling the pkgutil version.
+            for mod_info in pkgutil.iter_modules([path], prefix):
+                yield mod_info
+            continue
+
+        # START COPY OF pkutils._iter_file_finder_modules.
+        if importer.path is None or not os.path.isdir(importer.path):
+            return
+
+        yielded = {}
+
+        try:
+            filenames = os.listdir(importer.path)
+        except OSError:
+            # ignore unreadable directories like import does
+            filenames = []
+        filenames.sort()  # handle packages before same-named modules
+
+        for fn in filenames:
+            modname = inspect.getmodulename(fn)
+            if modname == '__init__' or modname in yielded:
+                continue
+
+            # jedi addition: Avoid traversing special directories
+            if fn.startswith('.') or fn == '__pycache__':
+                continue
+
+            path = os.path.join(importer.path, fn)
+            ispkg = False
+
+            if not modname and os.path.isdir(path) and '.' not in fn:
+                modname = fn
+                # A few jedi modifications: Don't check if there's an
+                # __init__.py
+                try:
+                    os.listdir(path)
+                except OSError:
+                    # ignore unreadable directories like import does
+                    continue
+                ispkg = True
+
+            if modname and '.' not in modname:
+                yielded[modname] = 1
+                yield importer, prefix + modname, ispkg
+        # END COPY
+
+
+iter_modules = _iter_modules if py_version >= 34 else pkgutil.iter_modules
+
+
+class ImplicitNSInfo(object):
+    """Stores information returned from an implicit namespace spec"""
+    def __init__(self, name, paths):
+        self.name = name
+        self.paths = paths
+
+
+if is_py3:
+    all_suffixes = importlib.machinery.all_suffixes
+else:
+    def all_suffixes():
+        # Is deprecated and raises a warning in Python 3.6.
+        import imp
+        return [suffix for suffix, _, _ in imp.get_suffixes()]
+
+
 # unicode function
 try:
     unicode = unicode
 except NameError:
     unicode = str
 
-if is_py3:
-    u = lambda s: s
-else:
-    u = lambda s: s.decode('utf-8')
-
-u.__doc__ = """
-Decode a raw string into unicode object.  Do nothing in Python 3.
-"""
-
-# exec function
-if is_py3:
-    def exec_function(source, global_map):
-        exec(source, global_map)
-else:
-    eval(compile("""def exec_function(source, global_map):
-                        exec source in global_map """, 'blub', 'exec'))
 
 # re-raise function
 if is_py3:
@@ -174,22 +290,12 @@ Usage::
 
 """
 
-class Python3Method(object):
-    def __init__(self, func):
-        self.func = func
-
-    def __get__(self, obj, objtype):
-        if obj is None:
-            return lambda *args, **kwargs: self.func(*args, **kwargs)
-        else:
-            return lambda *args, **kwargs: self.func(obj, *args, **kwargs)
-
 
 def use_metaclass(meta, *bases):
     """ Create a class with a metaclass. """
     if not bases:
         bases = (object,)
-    return meta("HackClass", bases, {})
+    return meta("Py2CompatibilityMetaClass", bases, {})
 
 
 try:
@@ -200,47 +306,70 @@ except AttributeError:
     encoding = 'ascii'
 
 
-def u(string):
+def u(string, errors='strict'):
     """Cast to unicode DAMMIT!
     Written because Python2 repr always implicitly casts to a string, so we
     have to cast back to a unicode (and we now that we always deal with valid
     unicode, because we check that in the beginning).
     """
-    if is_py3:
-        return str(string)
-
-    if not isinstance(string, unicode):
-        return unicode(str(string), 'UTF-8')
+    if isinstance(string, bytes):
+        return unicode(string, encoding='UTF-8', errors=errors)
     return string
 
+
+def cast_path(obj):
+    """
+    Take a bytes or str path and cast it to unicode.
+
+    Apparently it is perfectly fine to pass both byte and unicode objects into
+    the sys.path. This probably means that byte paths are normal at other
+    places as well.
+
+    Since this just really complicates everything and Python 2.7 will be EOL
+    soon anyway, just go with always strings.
+    """
+    return u(obj, errors='replace')
+
+
+def force_unicode(obj):
+    # Intentionally don't mix those two up, because those two code paths might
+    # be different in the future (maybe windows?).
+    return cast_path(obj)
+
+
 try:
     import builtins  # module name in python 3
 except ImportError:
-    import __builtin__ as builtins
+    import __builtin__ as builtins  # noqa: F401
 
 
-import ast
+import ast  # noqa: F401
 
 
 def literal_eval(string):
-    # py3.0, py3.1 and py32 don't support unicode literals. Support those, I
-    # don't want to write two versions of the tokenizer.
-    if is_py3 and sys.version_info.minor < 3:
-        if re.match('[uU][\'"]', string):
-            string = string[1:]
     return ast.literal_eval(string)
 
 
 try:
     from itertools import zip_longest
 except ImportError:
-    from itertools import izip_longest as zip_longest  # Python 2
+    from itertools import izip_longest as zip_longest  # Python 2  # noqa: F401
 
 try:
     FileNotFoundError = FileNotFoundError
 except NameError:
     FileNotFoundError = IOError
 
+try:
+    IsADirectoryError = IsADirectoryError
+except NameError:
+    IsADirectoryError = IOError
+
+try:
+    PermissionError = PermissionError
+except NameError:
+    PermissionError = IOError
+
 
 def no_unicode_pprint(dct):
     """
@@ -270,3 +399,267 @@ def utf8_repr(func):
         return func
     else:
         return wrapper
+
+
+if is_py3:
+    import queue
+else:
+    import Queue as queue  # noqa: F401
+
+try:
+    # Attempt to load the C implementation of pickle on Python 2 as it is way
+    # faster.
+    import cPickle as pickle
+except ImportError:
+    import pickle
+if sys.version_info[:2] == (3, 3):
+    """
+    Monkeypatch the unpickler in Python 3.3. This is needed, because the
+    argument `encoding='bytes'` is not supported in 3.3, but badly needed to
+    communicate with Python 2.
+    """
+
+    class NewUnpickler(pickle._Unpickler):
+        dispatch = dict(pickle._Unpickler.dispatch)
+
+        def _decode_string(self, value):
+            # Used to allow strings from Python 2 to be decoded either as
+            # bytes or Unicode strings.  This should be used only with the
+            # STRING, BINSTRING and SHORT_BINSTRING opcodes.
+            if self.encoding == "bytes":
+                return value
+            else:
+                return value.decode(self.encoding, self.errors)
+
+        def load_string(self):
+            data = self.readline()[:-1]
+            # Strip outermost quotes
+            if len(data) >= 2 and data[0] == data[-1] and data[0] in b'"\'':
+                data = data[1:-1]
+            else:
+                raise pickle.UnpicklingError("the STRING opcode argument must be quoted")
+            self.append(self._decode_string(pickle.codecs.escape_decode(data)[0]))
+        dispatch[pickle.STRING[0]] = load_string
+
+        def load_binstring(self):
+            # Deprecated BINSTRING uses signed 32-bit length
+            len, = pickle.struct.unpack('<i', self.read(4))
+            if len < 0:
+                raise pickle.UnpicklingError("BINSTRING pickle has negative byte count")
+            data = self.read(len)
+            self.append(self._decode_string(data))
+        dispatch[pickle.BINSTRING[0]] = load_binstring
+
+        def load_short_binstring(self):
+            len = self.read(1)[0]
+            data = self.read(len)
+            self.append(self._decode_string(data))
+        dispatch[pickle.SHORT_BINSTRING[0]] = load_short_binstring
+
+    def load(file, fix_imports=True, encoding="ASCII", errors="strict"):
+        return NewUnpickler(file, fix_imports=fix_imports,
+                            encoding=encoding, errors=errors).load()
+
+    def loads(s, fix_imports=True, encoding="ASCII", errors="strict"):
+        if isinstance(s, str):
+            raise TypeError("Can't load pickle from unicode string")
+        file = pickle.io.BytesIO(s)
+        return NewUnpickler(file, fix_imports=fix_imports,
+                            encoding=encoding, errors=errors).load()
+
+    pickle.Unpickler = NewUnpickler
+    pickle.load = load
+    pickle.loads = loads
+
+
+def pickle_load(file):
+    try:
+        if is_py3:
+            return pickle.load(file, encoding='bytes')
+        return pickle.load(file)
+    # Python on Windows don't throw EOF errors for pipes. So reraise them with
+    # the correct type, which is caught upwards.
+    except OSError:
+        if sys.platform == 'win32':
+            raise EOFError()
+        raise
+
+
+def _python2_dct_keys_to_unicode(data):
+    """
+    Python 2 stores object __dict__ entries as bytes, not unicode, correct it
+    here. Python 2 can deal with both, Python 3 expects unicode.
+    """
+    if isinstance(data, tuple):
+        return tuple(_python2_dct_keys_to_unicode(x) for x in data)
+    elif isinstance(data, list):
+        return list(_python2_dct_keys_to_unicode(x) for x in data)
+    elif hasattr(data, '__dict__') and type(data.__dict__) == dict:
+        data.__dict__ = {unicode(k): v for k, v in data.__dict__.items()}
+    return data
+
+
+def pickle_dump(data, file, protocol):
+    try:
+        if not is_py3:
+            data = _python2_dct_keys_to_unicode(data)
+        pickle.dump(data, file, protocol)
+        # On Python 3.3 flush throws sometimes an error even though the writing
+        # operation should be completed.
+        file.flush()
+    # Python on Windows don't throw EPIPE errors for pipes. So reraise them with
+    # the correct type and error number.
+    except OSError:
+        if sys.platform == 'win32':
+            raise IOError(errno.EPIPE, "Broken pipe")
+        raise
+
+
+# Determine the highest protocol version compatible for a given list of Python
+# versions.
+def highest_pickle_protocol(python_versions):
+    protocol = 4
+    for version in python_versions:
+        if version[0] == 2:
+            # The minimum protocol version for the versions of Python that we
+            # support (2.7 and 3.3+) is 2.
+            return 2
+        if version[1] < 4:
+            protocol = 3
+    return protocol
+
+
+try:
+    from inspect import Parameter
+except ImportError:
+    class Parameter(object):
+        POSITIONAL_ONLY = object()
+        POSITIONAL_OR_KEYWORD = object()
+        VAR_POSITIONAL = object()
+        KEYWORD_ONLY = object()
+        VAR_KEYWORD = object()
+
+
+class GeneralizedPopen(subprocess.Popen):
+    def __init__(self, *args, **kwargs):
+        if os.name == 'nt':
+            try:
+                # Was introduced in Python 3.7.
+                CREATE_NO_WINDOW = subprocess.CREATE_NO_WINDOW
+            except AttributeError:
+                CREATE_NO_WINDOW = 0x08000000
+            kwargs['creationflags'] = CREATE_NO_WINDOW
+        # The child process doesn't need file descriptors except 0, 1, 2.
+        # This is unix only.
+        kwargs['close_fds'] = 'posix' in sys.builtin_module_names
+        super(GeneralizedPopen, self).__init__(*args, **kwargs)
+
+
+# shutil.which is not available on Python 2.7.
+def which(cmd, mode=os.F_OK | os.X_OK, path=None):
+    """Given a command, mode, and a PATH string, return the path which
+    conforms to the given mode on the PATH, or None if there is no such
+    file.
+
+    `mode` defaults to os.F_OK | os.X_OK. `path` defaults to the result
+    of os.environ.get("PATH"), or can be overridden with a custom search
+    path.
+
+    """
+    # Check that a given file can be accessed with the correct mode.
+    # Additionally check that `file` is not a directory, as on Windows
+    # directories pass the os.access check.
+    def _access_check(fn, mode):
+        return (os.path.exists(fn) and os.access(fn, mode)
+                and not os.path.isdir(fn))
+
+    # If we're given a path with a directory part, look it up directly rather
+    # than referring to PATH directories. This includes checking relative to the
+    # current directory, e.g. ./script
+    if os.path.dirname(cmd):
+        if _access_check(cmd, mode):
+            return cmd
+        return None
+
+    if path is None:
+        path = os.environ.get("PATH", os.defpath)
+    if not path:
+        return None
+    path = path.split(os.pathsep)
+
+    if sys.platform == "win32":
+        # The current directory takes precedence on Windows.
+        if os.curdir not in path:
+            path.insert(0, os.curdir)
+
+        # PATHEXT is necessary to check on Windows.
+        pathext = os.environ.get("PATHEXT", "").split(os.pathsep)
+        # See if the given file matches any of the expected path extensions.
+        # This will allow us to short circuit when given "python.exe".
+        # If it does match, only test that one, otherwise we have to try
+        # others.
+        if any(cmd.lower().endswith(ext.lower()) for ext in pathext):
+            files = [cmd]
+        else:
+            files = [cmd + ext for ext in pathext]
+    else:
+        # On other platforms you don't have things like PATHEXT to tell you
+        # what file suffixes are executable, so just pass on cmd as-is.
+        files = [cmd]
+
+    seen = set()
+    for dir in path:
+        normdir = os.path.normcase(dir)
+        if normdir not in seen:
+            seen.add(normdir)
+            for thefile in files:
+                name = os.path.join(dir, thefile)
+                if _access_check(name, mode):
+                    return name
+    return None
+
+
+if not is_py3:
+    # Simplified backport of Python 3 weakref.finalize:
+    # https://github.com/python/cpython/blob/ded4737989316653469763230036b04513cb62b3/Lib/weakref.py#L502-L662
+    class finalize(object):
+        """Class for finalization of weakrefable objects.
+
+        finalize(obj, func, *args, **kwargs) returns a callable finalizer
+        object which will be called when obj is garbage collected. The
+        first time the finalizer is called it evaluates func(*arg, **kwargs)
+        and returns the result. After this the finalizer is dead, and
+        calling it just returns None.
+
+        When the program exits any remaining finalizers will be run.
+        """
+
+        # Finalizer objects don't have any state of their own.
+        # This ensures that they cannot be part of a ref-cycle.
+        __slots__ = ()
+        _registry = {}
+
+        def __init__(self, obj, func, *args, **kwargs):
+            info = functools.partial(func, *args, **kwargs)
+            info.weakref = weakref.ref(obj, self)
+            self._registry[self] = info
+
+        def __call__(self):
+            """Return func(*args, **kwargs) if alive."""
+            info = self._registry.pop(self, None)
+            if info:
+                return info()
+
+        @classmethod
+        def _exitfunc(cls):
+            if not cls._registry:
+                return
+            for finalizer in list(cls._registry):
+                try:
+                    finalizer()
+                except Exception:
+                    sys.excepthook(*sys.exc_info())
+                assert finalizer not in cls._registry
+
+    atexit.register(finalize._exitfunc)
+    weakref.finalize = finalize

jedi/api/__init__.py

@@ -10,44 +10,41 @@ arguments.
 .. warning:: Please, note that Jedi is **not thread safe**.
 """
 import os
-import warnings
 import sys
+import warnings
 
-from jedi.parser import load_grammar
-from jedi.parser import tree
-from jedi.parser.diff import FastParser
-from jedi.parser.utils import save_parser
+import parso
+from parso.python import tree
+
+from jedi._compatibility import force_unicode, cast_path, is_py3
+from jedi.parser_utils import get_executable_nodes
 from jedi import debug
 from jedi import settings
-from jedi import common
 from jedi import cache
+from jedi.file_io import KnownContentFileIO
 from jedi.api import classes
 from jedi.api import interpreter
-from jedi.api import usages
 from jedi.api import helpers
 from jedi.api.completion import Completion
+from jedi.api.environment import InterpreterEnvironment
+from jedi.api.project import get_default_project, Project
 from jedi.evaluate import Evaluator
-from jedi.evaluate import representation as er
 from jedi.evaluate import imports
-from jedi.evaluate.param import try_iter_content
-from jedi.evaluate.helpers import get_module_names
-from jedi.evaluate.sys_path import get_venv_path
-from jedi.evaluate.iterable import unpack_tuple_to_dict
-from jedi.evaluate.filters import TreeNameDefinition
+from jedi.evaluate import usages
+from jedi.evaluate.arguments import try_iter_content
+from jedi.evaluate.helpers import get_module_names, evaluate_call_of_leaf
+from jedi.evaluate.sys_path import transform_path_to_dotted
+from jedi.evaluate.names import TreeNameDefinition, ParamName
+from jedi.evaluate.syntax_tree import tree_name_to_contexts
+from jedi.evaluate.context import ModuleContext
+from jedi.evaluate.base_context import ContextSet
+from jedi.evaluate.context.iterable import unpack_tuple_to_dict
+from jedi.evaluate.gradual.conversion import convert_names, convert_contexts
+from jedi.evaluate.gradual.utils import load_proper_stub_module
 
 # Jedi uses lots and lots of recursion. By setting this a little bit higher, we
 # can remove some "maximum recursion depth" errors.
-sys.setrecursionlimit(2000)
-
-
-class NotFoundError(Exception):
-    """A custom error to avoid catching the wrong exceptions.
-
-    .. deprecated:: 0.9.0
-       Not in use anymore, Jedi just returns no goto result if you're not on a
-       valid name.
-    .. todo:: Remove!
-    """
+sys.setrecursionlimit(3000)
 
 
 class Script(object):
@@ -82,23 +79,14 @@ class Script(object):
     :param encoding: The encoding of ``source``, if it is not a
         ``unicode`` object (default ``'utf-8'``).
     :type encoding: str
-    :param source_encoding: The encoding of ``source``, if it is not a
-        ``unicode`` object (default ``'utf-8'``).
-    :type encoding: str
     :param sys_path: ``sys.path`` to use during analysis of the script
     :type sys_path: list
-
+    :param environment: TODO
+    :type environment: Environment
     """
     def __init__(self, source=None, line=None, column=None, path=None,
-                 encoding='utf-8', source_path=None, source_encoding=None,
-                 sys_path=None):
-        if source_path is not None:
-            warnings.warn("Use path instead of source_path.", DeprecationWarning)
-            path = source_path
-        if source_encoding is not None:
-            warnings.warn("Use encoding instead of source_encoding.", DeprecationWarning)
-            encoding = source_encoding
-
+                 encoding='utf-8', sys_path=None, environment=None,
+                 _project=None):
         self._orig_path = path
         # An empty path (also empty string) should always result in no path.
         self.path = os.path.abspath(path) if path else None
@@ -108,55 +96,108 @@ class Script(object):
             with open(path, 'rb') as f:
                 source = f.read()
 
-        self._source = common.source_to_unicode(source, encoding)
-        self._code_lines = common.splitlines(self._source)
+        # Load the Python grammar of the current interpreter.
+        self._grammar = parso.load_grammar()
+
+        if sys_path is not None and not is_py3:
+            sys_path = list(map(force_unicode, sys_path))
+
+        project = _project
+        if project is None:
+            # Load the Python grammar of the current interpreter.
+            project = get_default_project(
+                os.path.dirname(self.path)if path else os.getcwd()
+            )
+        # TODO deprecate and remove sys_path from the Script API.
+        if sys_path is not None:
+            project._sys_path = sys_path
+        self._evaluator = Evaluator(
+            project, environment=environment, script_path=self.path
+        )
+        debug.speed('init')
+        self._module_node, source = self._evaluator.parse_and_get_code(
+            code=source,
+            path=self.path,
+            encoding=encoding,
+            use_latest_grammar=path and path.endswith('.pyi'),
+            cache=False,  # No disk cache, because the current script often changes.
+            diff_cache=settings.fast_parser,
+            cache_path=settings.cache_directory,
+        )
+        debug.speed('parsed')
+        self._code_lines = parso.split_lines(source, keepends=True)
+        self._code = source
         line = max(len(self._code_lines), 1) if line is None else line
         if not (0 < line <= len(self._code_lines)):
             raise ValueError('`line` parameter is not in a valid range.')
 
-        line_len = len(self._code_lines[line - 1])
+        line_string = self._code_lines[line - 1]
+        line_len = len(line_string)
+        if line_string.endswith('\r\n'):
+            line_len -= 1
+        if line_string.endswith('\n'):
+            line_len -= 1
+
         column = line_len if column is None else column
         if not (0 <= column <= line_len):
-            raise ValueError('`column` parameter is not in a valid range.')
+            raise ValueError('`column` parameter (%d) is not in a valid range '
+                             '(0-%d) for line %d (%r).' % (
+                                 column, line_len, line, line_string))
         self._pos = line, column
-        self._path = path
 
         cache.clear_time_caches()
         debug.reset_time()
-        self._grammar = load_grammar(version='%s.%s' % sys.version_info[:2])
-        if sys_path is None:
-            venv = os.getenv('VIRTUAL_ENV')
-            if venv:
-                sys_path = list(get_venv_path(venv))
-        self._evaluator = Evaluator(self._grammar, sys_path=sys_path)
-        debug.speed('init')
-
-    @cache.memoize_method
-    def _get_module_node(self):
-        cache.invalidate_star_import_cache(self._path)
-        parser = FastParser(self._grammar, self._source, self.path)
-        save_parser(self.path, parser, pickling=False)
-
-        return parser.module
 
+    # Cache the module, this is mostly useful for testing, since this shouldn't
+    # be called multiple times.
     @cache.memoize_method
     def _get_module(self):
-        module = er.ModuleContext(self._evaluator, self._get_module_node())
-        imports.add_module(self._evaluator, module.name.string_name, module)
+        names = None
+        is_package = False
+        if self.path is not None:
+            import_names, is_p = transform_path_to_dotted(
+                self._evaluator.get_sys_path(add_parent_paths=False),
+                self.path
+            )
+            if import_names is not None:
+                names = import_names
+                is_package = is_p
+
+        if self.path is None:
+            file_io = None
+        else:
+            file_io = KnownContentFileIO(cast_path(self.path), self._code)
+        if self.path is not None and self.path.endswith('.pyi'):
+            # We are in a stub file. Try to load the stub properly.
+            stub_module = load_proper_stub_module(
+                self._evaluator,
+                file_io,
+                names,
+                self._module_node
+            )
+            if stub_module is not None:
+                return stub_module
+
+        if names is None:
+            names = ('__main__',)
+
+        module = ModuleContext(
+            self._evaluator, self._module_node, file_io,
+            string_names=names,
+            code_lines=self._code_lines,
+            is_package=is_package,
+        )
+        if names[0] not in ('builtins', '__builtin__', 'typing'):
+            # These modules are essential for Jedi, so don't overwrite them.
+            self._evaluator.module_cache.add(names, ContextSet([module]))
         return module
 
-    @property
-    def source_path(self):
-        """
-        .. deprecated:: 0.7.0
-           Use :attr:`.path` instead.
-        .. todo:: Remove!
-        """
-        warnings.warn("Use path instead of source_path.", DeprecationWarning)
-        return self.path
-
     def __repr__(self):
-        return '<%s: %s>' % (self.__class__.__name__, repr(self._orig_path))
+        return '<%s: %s %r>' % (
+            self.__class__.__name__,
+            repr(self._orig_path),
+            self._evaluator.environment,
+        )
 
     def completions(self):
         """
@@ -166,16 +207,14 @@ class Script(object):
         :return: Completion objects, sorted by name and __ comes last.
         :rtype: list of :class:`classes.Completion`
         """
-        debug.speed('completions start')
-        completion = Completion(
-            self._evaluator, self._get_module(), self._code_lines,
-            self._pos, self.call_signatures
-        )
-        completions = completion.completions()
-        debug.speed('completions end')
-        return completions
+        with debug.increase_indent_cm('completions'):
+            completion = Completion(
+                self._evaluator, self._get_module(), self._code_lines,
+                self._pos, self.call_signatures
+            )
+            return completion.completions()
 
-    def goto_definitions(self):
+    def goto_definitions(self, **kwargs):
         """
         Return the definitions of a the path under the cursor.  goto function!
         This follows complicated paths and returns the end, not the first
@@ -185,60 +224,96 @@ class Script(object):
         because Python itself is a dynamic language, which means depending on
         an option you can have two different versions of a function.
 
+        :param only_stubs: Only return stubs for this goto call.
+        :param prefer_stubs: Prefer stubs to Python objects for this type
+            inference call.
         :rtype: list of :class:`classes.Definition`
         """
-        module_node = self._get_module_node()
-        leaf = module_node.name_for_position(self._pos)
+        with debug.increase_indent_cm('goto_definitions'):
+            return self._goto_definitions(**kwargs)
+
+    def _goto_definitions(self, only_stubs=False, prefer_stubs=False):
+        leaf = self._module_node.get_name_of_position(self._pos)
         if leaf is None:
-            leaf = module_node.get_leaf_for_position(self._pos)
+            leaf = self._module_node.get_leaf_for_position(self._pos)
             if leaf is None:
                 return []
 
         context = self._evaluator.create_context(self._get_module(), leaf)
-        definitions = helpers.evaluate_goto_definition(self._evaluator, context, leaf)
 
-        names = [s.name for s in definitions]
-        defs = [classes.Definition(self._evaluator, name) for name in names]
+        contexts = helpers.evaluate_goto_definition(self._evaluator, context, leaf)
+        contexts = convert_contexts(
+            contexts,
+            only_stubs=only_stubs,
+            prefer_stubs=prefer_stubs,
+        )
+
+        defs = [classes.Definition(self._evaluator, c.name) for c in contexts]
         # The additional set here allows the definitions to become unique in an
         # API sense. In the internals we want to separate more things than in
         # the API.
         return helpers.sorted_definitions(set(defs))
 
-    def goto_assignments(self, follow_imports=False):
+    def goto_assignments(self, follow_imports=False, follow_builtin_imports=False, **kwargs):
         """
         Return the first definition found, while optionally following imports.
         Multiple objects may be returned, because Python itself is a
         dynamic language, which means depending on an option you can have two
         different versions of a function.
 
+        .. note:: It is deprecated to use follow_imports and follow_builtin_imports as
+            positional arguments. Will be a keyword argument in 0.16.0.
+
+        :param follow_imports: The goto call will follow imports.
+        :param follow_builtin_imports: If follow_imports is True will decide if
+            it follow builtin imports.
+        :param only_stubs: Only return stubs for this goto call.
+        :param prefer_stubs: Prefer stubs to Python objects for this goto call.
         :rtype: list of :class:`classes.Definition`
         """
-        def filter_follow_imports(names):
+        with debug.increase_indent_cm('goto_assignments'):
+            return self._goto_assignments(follow_imports, follow_builtin_imports, **kwargs)
+
+    def _goto_assignments(self, follow_imports, follow_builtin_imports,
+                          only_stubs=False, prefer_stubs=False):
+        def filter_follow_imports(names, check):
             for name in names:
-                if isinstance(name, (imports.ImportName, TreeNameDefinition)):
-                    for context in name.infer():
-                        yield context.name
+                if check(name):
+                    new_names = list(filter_follow_imports(name.goto(), check))
+                    found_builtin = False
+                    if follow_builtin_imports:
+                        for new_name in new_names:
+                            if new_name.start_pos is None:
+                                found_builtin = True
+
+                    if found_builtin:
+                        yield name
+                    else:
+                        for new_name in new_names:
+                            yield new_name
                 else:
                     yield name
 
-        names = self._goto()
+        tree_name = self._module_node.get_name_of_position(self._pos)
+        if tree_name is None:
+            # Without a name we really just want to jump to the result e.g.
+            # executed by `foo()`, if we the cursor is after `)`.
+            return self.goto_definitions(only_stubs=only_stubs, prefer_stubs=prefer_stubs)
+        context = self._evaluator.create_context(self._get_module(), tree_name)
+        names = list(self._evaluator.goto(context, tree_name))
+
         if follow_imports:
-            names = filter_follow_imports(names)
+            names = filter_follow_imports(names, lambda name: name.is_import())
+        names = convert_names(
+            names,
+            only_stubs=only_stubs,
+            prefer_stubs=prefer_stubs,
+        )
 
         defs = [classes.Definition(self._evaluator, d) for d in set(names)]
         return helpers.sorted_definitions(defs)
 
-    def _goto(self):
-        """
-        Used for goto_assignments and usages.
-        """
-        name = self._get_module_node().name_for_position(self._pos)
-        if name is None:
-            return []
-        context = self._evaluator.create_context(self._get_module(), name)
-        return list(self._evaluator.goto(context, name))
-
-    def usages(self, additional_module_paths=()):
+    def usages(self, additional_module_paths=(), **kwargs):
         """
         Return :class:`classes.Definition` objects, which contain all
         names that point to the definition of the name under the cursor. This
@@ -247,38 +322,31 @@ class Script(object):
 
         .. todo:: Implement additional_module_paths
 
+        :param additional_module_paths: Deprecated, never ever worked.
+        :param include_builtins: Default True, checks if a usage is a builtin
+            (e.g. ``sys``) and in that case does not return it.
         :rtype: list of :class:`classes.Definition`
         """
-        temp, settings.dynamic_flow_information = \
-            settings.dynamic_flow_information, False
-        try:
-            module_node = self._get_module_node()
-            user_stmt = module_node.get_statement_for_position(self._pos)
-            definition_names = self._goto()
-            if not definition_names and isinstance(user_stmt, tree.Import):
-                # For not defined imports (goto doesn't find something, we take
-                # the name as a definition. This is enough, because every name
-                # points to it.
-                name = user_stmt.name_for_position(self._pos)
-                if name is None:
-                    # Must be syntax
-                    return []
-                definition_names = [TreeNameDefinition(self._get_module(), name)]
+        if additional_module_paths:
+            warnings.warn(
+                "Deprecated since version 0.12.0. This never even worked, just ignore it.",
+                DeprecationWarning,
+                stacklevel=2
+            )
 
-            if not definition_names:
-                # Without a definition for a name we cannot find references.
+        def _usages(include_builtins=True):
+            tree_name = self._module_node.get_name_of_position(self._pos)
+            if tree_name is None:
+                # Must be syntax
                 return []
 
-            definition_names = usages.resolve_potential_imports(self._evaluator,
-                                                                definition_names)
+            names = usages.usages(self._get_module(), tree_name)
 
-            modules = set([d.get_root_context() for d in definition_names])
-            modules.add(self._get_module())
-            definitions = usages.usages(self._evaluator, definition_names, modules)
-        finally:
-            settings.dynamic_flow_information = temp
-
-        return helpers.sorted_definitions(set(definitions))
+            definitions = [classes.Definition(self._evaluator, n) for n in names]
+            if not include_builtins:
+                definitions = [d for d in definitions if not d.in_builtin_module()]
+            return helpers.sorted_definitions(definitions)
+        return _usages(**kwargs)
 
     def call_signatures(self):
         """
@@ -296,56 +364,55 @@ class Script(object):
 
         :rtype: list of :class:`classes.CallSignature`
         """
-        call_signature_details = \
-            helpers.get_call_signature_details(self._get_module_node(), self._pos)
-        if call_signature_details is None:
+        call_details = helpers.get_call_signature_details(self._module_node, self._pos)
+        if call_details is None:
             return []
 
         context = self._evaluator.create_context(
             self._get_module(),
-            call_signature_details.bracket_leaf
+            call_details.bracket_leaf
+        )
+        definitions = helpers.cache_call_signatures(
+            self._evaluator,
+            context,
+            call_details.bracket_leaf,
+            self._code_lines,
+            self._pos
         )
-        with common.scale_speed_settings(settings.scale_call_signatures):
-            definitions = helpers.cache_call_signatures(
-                self._evaluator,
-                context,
-                call_signature_details.bracket_leaf,
-                self._code_lines,
-                self._pos
-            )
         debug.speed('func_call followed')
 
-        return [classes.CallSignature(self._evaluator, d.name,
-                                      call_signature_details.bracket_leaf.start_pos,
-                                      call_signature_details.call_index,
-                                      call_signature_details.keyword_name_str)
-                for d in definitions if hasattr(d, 'py__call__')]
+        # TODO here we use stubs instead of the actual contexts. We should use
+        # the signatures from stubs, but the actual contexts, probably?!
+        return [classes.CallSignature(self._evaluator, signature, call_details)
+                for signature in definitions.get_signatures()]
 
     def _analysis(self):
         self._evaluator.is_analysis = True
-        module_node = self._get_module_node()
-        self._evaluator.analysis_modules = [module_node]
+        self._evaluator.analysis_modules = [self._module_node]
+        module = self._get_module()
         try:
-            for node in module_node.nodes_to_execute():
-                context = self._get_module().create_context(node)
+            for node in get_executable_nodes(self._module_node):
+                context = module.create_context(node)
                 if node.type in ('funcdef', 'classdef'):
-                    # TODO This is stupid, should be private
-                    from jedi.evaluate.finder import _name_to_types
                     # Resolve the decorators.
-                    _name_to_types(self._evaluator, context, node.children[1])
+                    tree_name_to_contexts(self._evaluator, context, node.children[1])
                 elif isinstance(node, tree.Import):
                     import_names = set(node.get_defined_names())
                     if node.is_nested():
-                        import_names |= set(path[-1] for path in node.paths())
+                        import_names |= set(path[-1] for path in node.get_paths())
                     for n in import_names:
                         imports.infer_import(context, n)
                 elif node.type == 'expr_stmt':
                     types = context.eval_node(node)
                     for testlist in node.children[:-1:2]:
                         # Iterate tuples.
-                        unpack_tuple_to_dict(self._evaluator, types, testlist)
+                        unpack_tuple_to_dict(context, types, testlist)
                 else:
-                    try_iter_content(self._evaluator.goto_definitions(context, node))
+                    if node.type == 'name':
+                        defs = self._evaluator.goto_definitions(context, node)
+                    else:
+                        defs = evaluate_call_of_leaf(context, node)
+                    try_iter_content(defs)
                 self._evaluator.reset_recursion_limitations()
 
             ana = [a for a in self._evaluator.analysis if self.path == a.path]
@@ -369,6 +436,7 @@ class Interpreter(Script):
     >>> print(script.completions()[0].name)
     upper
     """
+    _allow_descriptor_getattr_default = True
 
     def __init__(self, source, namespaces, **kwds):
         """
@@ -389,40 +457,30 @@ class Interpreter(Script):
         except Exception:
             raise TypeError("namespaces must be a non-empty list of dicts.")
 
-        super(Interpreter, self).__init__(source, **kwds)
+        environment = kwds.get('environment', None)
+        if environment is None:
+            environment = InterpreterEnvironment()
+        else:
+            if not isinstance(environment, InterpreterEnvironment):
+                raise TypeError("The environment needs to be an InterpreterEnvironment subclass.")
+
+        super(Interpreter, self).__init__(source, environment=environment,
+                                          _project=Project(os.getcwd()), **kwds)
         self.namespaces = namespaces
+        self._evaluator.allow_descriptor_getattr = self._allow_descriptor_getattr_default
 
     def _get_module(self):
-        parser_module = super(Interpreter, self)._get_module_node()
         return interpreter.MixedModuleContext(
             self._evaluator,
-            parser_module,
-            self.namespaces
+            self._module_node,
+            self.namespaces,
+            file_io=KnownContentFileIO(self.path, self._code),
+            code_lines=self._code_lines,
         )
 
 
-def defined_names(source, path=None, encoding='utf-8'):
-    """
-    Get all definitions in `source` sorted by its position.
-
-    This functions can be used for listing functions, classes and
-    data defined in a file.  This can be useful if you want to list
-    them in "sidebar".  Each element in the returned list also has
-    `defined_names` method which can be used to get sub-definitions
-    (e.g., methods in class).
-
-    :rtype: list of classes.Definition
-
-    .. deprecated:: 0.9.0
-       Use :func:`names` instead.
-    .. todo:: Remove!
-    """
-    warnings.warn("Use call_signatures instead.", DeprecationWarning)
-    return names(source, path, encoding)
-
-
 def names(source=None, path=None, encoding='utf-8', all_scopes=False,
-          definitions=True, references=False):
+          definitions=True, references=False, environment=None):
     """
     Returns a list of `Definition` objects, containing name parts.
     This means you can call ``Definition.goto_assignments()`` and get the
@@ -441,17 +499,24 @@ def names(source=None, path=None, encoding='utf-8', all_scopes=False,
         is_def = _def._name.tree_name.is_definition()
         return definitions and is_def or references and not is_def
 
+    def create_name(name):
+        if name.parent.type == 'param':
+            cls = ParamName
+        else:
+            cls = TreeNameDefinition
+        return cls(
+            module_context.create_context(name),
+            name
+        )
+
     # Set line/column to a random position, because they don't matter.
-    script = Script(source, line=1, column=0, path=path, encoding=encoding)
+    script = Script(source, line=1, column=0, path=path, encoding=encoding, environment=environment)
     module_context = script._get_module()
     defs = [
         classes.Definition(
             script._evaluator,
-            TreeNameDefinition(
-                module_context.create_context(name.parent),
-                name
-            )
-        ) for name in get_module_names(script._get_module_node(), all_scopes)
+            create_name(name)
+        ) for name in get_module_names(script._module_node, all_scopes)
     ]
     return sorted(filter(def_ref_filter, defs), key=lambda x: (x.line, x.column))
 

jedi/api/classes.py

@@ -3,19 +3,21 @@ The :mod:`jedi.api.classes` module contains the return classes of the API.
 These classes are the much bigger part of the whole API, because they contain
 the interesting information about completion and goto operations.
 """
-import warnings
 import re
+import sys
+import warnings
 
-from jedi._compatibility import u
 from jedi import settings
-from jedi import common
-from jedi.parser.utils import load_parser
+from jedi import debug
+from jedi.evaluate.utils import unite
 from jedi.cache import memoize_method
-from jedi.evaluate import representation as er
-from jedi.evaluate import instance
 from jedi.evaluate import imports
 from jedi.evaluate import compiled
-from jedi.evaluate.filters import ParamName
+from jedi.evaluate.imports import ImportName
+from jedi.evaluate.context import FunctionExecutionContext
+from jedi.evaluate.gradual.typeshed import StubModuleContext
+from jedi.evaluate.gradual.conversion import convert_names, convert_contexts
+from jedi.evaluate.base_context import ContextSet
 from jedi.api.keywords import KeywordName
 
 
@@ -35,6 +37,10 @@ def defined_names(evaluator, context):
     return [Definition(evaluator, n) for n in _sort_names_by_start_pos(names)]
 
 
+def _contexts_to_definitions(contexts):
+    return [Definition(c.evaluator, c.name) for c in contexts]
+
+
 class BaseDefinition(object):
     _mapping = {
         'posixpath': 'os.path',
@@ -46,9 +52,10 @@ class BaseDefinition(object):
         'posix': 'os',
         '_io': 'io',
         '_functools': 'functools',
+        '_collections': 'collections',
+        '_socket': 'socket',
         '_sqlite3': 'sqlite3',
-        '__builtin__': '',
-        'builtins': '',
+        '__builtin__': 'builtins',
     }
 
     _tuple_mapping = dict((tuple(k.split('.')), v) for (k, v) in {
@@ -59,17 +66,27 @@ class BaseDefinition(object):
         self._evaluator = evaluator
         self._name = name
         """
-        An instance of :class:`jedi.parser.reprsentation.Name` subclass.
+        An instance of :class:`parso.python.tree.Name` subclass.
         """
         self.is_keyword = isinstance(self._name, KeywordName)
 
-        # generate a path to the definition
-        self._module = name.get_root_context()
-        if self.in_builtin_module():
-            self.module_path = None
-        else:
-            self.module_path = self._module.py__file__()
-            """Shows the file path of a module. e.g. ``/usr/lib/python2.7/os.py``"""
+    @memoize_method
+    def _get_module(self):
+        # This can take a while to complete, because in the worst case of
+        # imports (consider `import a` completions), we need to load all
+        # modules starting with a first.
+        return self._name.get_root_context()
+
+    @property
+    def module_path(self):
+        """Shows the file path of a module. e.g. ``/usr/lib/python2.7/os.py``"""
+        module = self._get_module()
+        if module.is_stub() or not module.is_compiled():
+            # Compiled modules should not return a module path even if they
+            # have one.
+            return self._get_module().py__file__()
+
+        return None
 
     @property
     def name(self):
@@ -92,6 +109,7 @@ class BaseDefinition(object):
         to Jedi, :meth:`jedi.Script.goto_definitions` should return a list of
         definition for ``sys``, ``f``, ``C`` and ``x``.
 
+        >>> from jedi._compatibility import no_unicode_pprint
         >>> from jedi import Script
         >>> source = '''
         ... import keyword
@@ -117,29 +135,35 @@ class BaseDefinition(object):
         so that it is easy to relate the result to the source code.
 
         >>> defs = sorted(defs, key=lambda d: d.line)
-        >>> defs                           # doctest: +NORMALIZE_WHITESPACE
-        [<Definition module keyword>, <Definition class C>,
-         <Definition instance D>, <Definition def f>]
+        >>> no_unicode_pprint(defs)  # doctest: +NORMALIZE_WHITESPACE
+        [<Definition full_name='keyword', description='module keyword'>,
+         <Definition full_name='__main__.C', description='class C'>,
+         <Definition full_name='__main__.D', description='instance D'>,
+         <Definition full_name='__main__.f', description='def f'>]
 
         Finally, here is what you can get from :attr:`type`:
 
-        >>> defs[0].type
+        >>> defs = [str(d.type) for d in defs]  # It's unicode and in Py2 has u before it.
+        >>> defs[0]
         'module'
-        >>> defs[1].type
+        >>> defs[1]
         'class'
-        >>> defs[2].type
+        >>> defs[2]
         'instance'
-        >>> defs[3].type
+        >>> defs[3]
         'function'
 
+        Valid values for are ``module``, ``class``, ``instance``, ``function``,
+        ``param``, ``path`` and ``keyword``.
+
         """
         tree_name = self._name.tree_name
         resolve = False
         if tree_name is not None:
             # TODO move this to their respective names.
             definition = tree_name.get_definition()
-            if definition.type == 'import_from' and \
-                    tree_name in definition.get_defined_names():
+            if definition is not None and definition.type == 'import_from' and \
+                    tree_name.is_definition():
                 resolve = True
 
         if isinstance(self._name, imports.SubModuleName) or resolve:
@@ -147,38 +171,6 @@ class BaseDefinition(object):
                 return context.api_type
         return self._name.api_type
 
-    def _path(self):
-        """The path to a module/class/function definition."""
-        def to_reverse():
-            name = self._name
-            if name.api_type == 'module':
-                try:
-                    name = list(name.infer())[0].name
-                except IndexError:
-                    pass
-
-            if name.api_type == 'module':
-                module_context, = name.infer()
-                for n in reversed(module_context.py__name__().split('.')):
-                    yield n
-            else:
-                yield name.string_name
-
-            parent_context = name.parent_context
-            while parent_context is not None:
-                try:
-                    method = parent_context.py__name__
-                except AttributeError:
-                    try:
-                        yield parent_context.name.string_name
-                    except AttributeError:
-                        pass
-                else:
-                    for name in reversed(method().split('.')):
-                        yield name
-                parent_context = parent_context.parent_context
-        return reversed(list(to_reverse()))
-
     @property
     def module_name(self):
         """
@@ -188,14 +180,17 @@ class BaseDefinition(object):
         >>> source = 'import json'
         >>> script = Script(source, path='example.py')
         >>> d = script.goto_definitions()[0]
-        >>> print(d.module_name)                       # doctest: +ELLIPSIS
+        >>> print(d.module_name)  # doctest: +ELLIPSIS
         json
         """
-        return self._module.name.string_name
+        return self._get_module().name.string_name
 
     def in_builtin_module(self):
         """Whether this is a builtin module."""
-        return isinstance(self._module, compiled.CompiledObject)
+        if isinstance(self._get_module(), StubModuleContext):
+            return any(isinstance(context, compiled.CompiledObject)
+                       for context in self._get_module().non_stub_context_set)
+        return isinstance(self._get_module(), compiled.CompiledObject)
 
     @property
     def line(self):
@@ -244,35 +239,12 @@ class BaseDefinition(object):
             the ``foo.docstring(fast=False)`` on every object, because it
             parses all libraries starting with ``a``.
         """
-        if raw:
-            return _Help(self._name).raw(fast=fast)
-        else:
-            return _Help(self._name).full(fast=fast)
-
-    @property
-    def doc(self):
-        """
-        .. deprecated:: 0.8.0
-           Use :meth:`.docstring` instead.
-        .. todo:: Remove!
-        """
-        warnings.warn("Use docstring() instead.", DeprecationWarning)
-        return self.docstring()
-
-    @property
-    def raw_doc(self):
-        """
-        .. deprecated:: 0.8.0
-           Use :meth:`.docstring` instead.
-        .. todo:: Remove!
-        """
-        warnings.warn("Use docstring() instead.", DeprecationWarning)
-        return self.docstring(raw=True)
+        return _Help(self._name).docstring(fast=fast, raw=raw)
 
     @property
     def description(self):
         """A textual description of the object."""
-        return u(self._name.string_name)
+        return self._name.string_name
 
     @property
     def full_name(self):
@@ -298,84 +270,111 @@ class BaseDefinition(object):
         be ``<module 'posixpath' ...>```. However most users find the latter
         more practical.
         """
-        path = list(self._path())
-        # TODO add further checks, the mapping should only occur on stdlib.
-        if not path:
-            return None  # for keywords the path is empty
+        if not self._name.is_context_name:
+            return None
 
-        with common.ignored(KeyError):
-            path[0] = self._mapping[path[0]]
-        for key, repl in self._tuple_mapping.items():
-            if tuple(path[:len(key)]) == key:
-                path = [repl] + path[len(key):]
+        names = self._name.get_qualified_names(include_module_names=True)
+        if names is None:
+            return names
 
-        return '.'.join(path if path[0] else path[1:])
+        names = list(names)
+        try:
+            names[0] = self._mapping[names[0]]
+        except KeyError:
+            pass
 
-    def goto_assignments(self):
-        if self._name.tree_name is None:
-            return self
+        return '.'.join(names)
 
-        names = self._evaluator.goto(self._name.parent_context, self._name.tree_name)
-        return [Definition(self._evaluator, n) for n in names]
+    def is_stub(self):
+        if not self._name.is_context_name:
+            return False
 
-    def _goto_definitions(self):
-        # TODO make this function public.
-        return [Definition(self._evaluator, d.name) for d in self._name.infer()]
+        return self._name.get_root_context().is_stub()
+
+    def goto_assignments(self, **kwargs):  # Python 2...
+        with debug.increase_indent_cm('goto for %s' % self._name):
+            return self._goto_assignments(**kwargs)
+
+    def _goto_assignments(self, only_stubs=False, prefer_stubs=False):
+        assert not (only_stubs and prefer_stubs)
+
+        if not self._name.is_context_name:
+            return []
+
+        names = convert_names(
+            self._name.goto(),
+            only_stubs=only_stubs,
+            prefer_stubs=prefer_stubs,
+        )
+        return [self if n == self._name else Definition(self._evaluator, n)
+                for n in names]
+
+    def infer(self, **kwargs):  # Python 2...
+        with debug.increase_indent_cm('infer for %s' % self._name):
+            return self._infer(**kwargs)
+
+    def _infer(self, only_stubs=False, prefer_stubs=False):
+        assert not (only_stubs and prefer_stubs)
+
+        if not self._name.is_context_name:
+            return []
+
+        # First we need to make sure that we have stub names (if possible) that
+        # we can follow. If we don't do that, we can end up with the inferred
+        # results of Python objects instead of stubs.
+        names = convert_names([self._name], prefer_stubs=True)
+        contexts = convert_contexts(
+            ContextSet.from_sets(n.infer() for n in names),
+            only_stubs=only_stubs,
+            prefer_stubs=prefer_stubs,
+        )
+        resulting_names = [c.name for c in contexts]
+        return [self if n == self._name else Definition(self._evaluator, n)
+                for n in resulting_names]
 
     @property
     @memoize_method
     def params(self):
         """
-        Raises an ``AttributeError``if the definition is not callable.
+        Deprecated! Will raise a warning soon. Use get_signatures()[...].params.
+
+        Raises an ``AttributeError`` if the definition is not callable.
         Otherwise returns a list of `Definition` that represents the params.
         """
-        def get_param_names(context):
-            param_names = []
-            if context.api_type == 'function':
-                param_names = list(context.get_param_names())
-                if isinstance(context, instance.BoundMethod):
-                    param_names = param_names[1:]
-            elif isinstance(context, (instance.AbstractInstanceContext, er.ClassContext)):
-                if isinstance(context, er.ClassContext):
-                    search = '__init__'
-                else:
-                    search = '__call__'
-                names = context.get_function_slot_names(search)
-                if not names:
-                    return []
+        # Only return the first one. There might be multiple one, especially
+        # with overloading.
+        for context in self._name.infer():
+            for signature in context.get_signatures():
+                return [
+                    Definition(self._evaluator, n)
+                    for n in signature.get_param_names(resolve_stars=True)
+                ]
 
-                # Just take the first one here, not optimal, but currently
-                # there's no better solution.
-                inferred = names[0].infer()
-                param_names = get_param_names(next(iter(inferred)))
-                if isinstance(context, er.ClassContext):
-                    param_names = param_names[1:]
-                return param_names
-            elif isinstance(context, compiled.CompiledObject):
-                return list(context.get_param_names())
-            return param_names
-
-        followed = list(self._name.infer())
-        if not followed or not hasattr(followed[0], 'py__call__'):
-            raise AttributeError()
-        context = followed[0]  # only check the first one.
-
-        return [_Param(self._evaluator, n) for n in get_param_names(context)]
+        if self.type == 'function' or self.type == 'class':
+            # Fallback, if no signatures were defined (which is probably by
+            # itself a bug).
+            return []
+        raise AttributeError('There are no params defined on this.')
 
     def parent(self):
+        if not self._name.is_context_name:
+            return None
+
         context = self._name.parent_context
         if context is None:
             return None
 
-        if isinstance(context, er.FunctionExecutionContext):
-            # TODO the function context should be a part of the function
-            # execution context.
-            context = er.FunctionContext(
-                self._evaluator, context.parent_context, context.tree_node)
+        if isinstance(context, FunctionExecutionContext):
+            context = context.function_context
         return Definition(self._evaluator, context.name)
 
     def __repr__(self):
-        return "<%s %s>" % (type(self).__name__, self.description)
+        return "<%s %sname=%r, description=%r>" % (
+            self.__class__.__name__,
+            'full_' if self.full_name else '',
+            self.full_name or self.name,
+            self.description,
+        )
 
     def get_line_code(self, before=0, after=0):
         """
@@ -387,16 +386,20 @@ class BaseDefinition(object):
         :return str: Returns the line(s) of code or an empty string if it's a
                      builtin.
         """
-        if self.in_builtin_module():
+        if not self._name.is_context_name or self.in_builtin_module():
             return ''
 
-        path = self._name.get_root_context().py__file__()
-        parser = load_parser(path)
-        lines = common.splitlines(parser.source)
+        lines = self._name.get_root_context().code_lines
 
-        line_nr = self._name.start_pos[0]
-        start_line_nr = line_nr - before
-        return '\n'.join(lines[start_line_nr:line_nr + after + 1])
+        index = self._name.start_pos[0] - 1
+        start_index = max(index - before, 0)
+        return ''.join(lines[start_index:index + after + 1])
+
+    def get_signatures(self):
+        return [Signature(self._evaluator, s) for s in self._name.infer().get_signatures()]
+
+    def execute(self):
+        return _contexts_to_definitions(self._name.infer().execute_evaluated())
 
 
 class Completion(BaseDefinition):
@@ -417,12 +420,13 @@ class Completion(BaseDefinition):
     def _complete(self, like_name):
         append = ''
         if settings.add_bracket_after_function \
-                and self.type == 'Function':
+                and self.type == 'function':
             append = '('
 
-        if isinstance(self._name, ParamName) and self._stack is not None:
-            node_names = list(self._stack.get_node_names(self._evaluator.grammar))
-            if 'trailer' in node_names and 'argument' not in node_names:
+        if self._name.api_type == 'param' and self._stack is not None:
+            nonterminals = [stack_node.nonterminal for stack_node in self._stack]
+            if 'trailer' in nonterminals and 'argument' not in nonterminals:
+                # TODO this doesn't work for nested calls.
                 append += '='
 
         name = self._name.string_name
@@ -472,7 +476,7 @@ class Completion(BaseDefinition):
             # In this case we can just resolve the like name, because we
             # wouldn't load like > 100 Python modules anymore.
             fast = False
-        return super(Completion, self,).docstring(raw, fast)
+        return super(Completion, self).docstring(raw=raw, fast=fast)
 
     @property
     def description(self):
@@ -486,6 +490,8 @@ class Completion(BaseDefinition):
     @memoize_method
     def follow_definition(self):
         """
+        Deprecated!
+
         Return the original definitions. I strongly recommend not using it for
         your completions, because it might slow down |jedi|. If you want to
         read only a few objects (<=20), it might be useful, especially to get
@@ -493,8 +499,12 @@ class Completion(BaseDefinition):
         follows all results. This means with 1000 completions (e.g.  numpy),
         it's just PITA-slow.
         """
-        defs = self._name.infer()
-        return [Definition(self._evaluator, d.name) for d in defs]
+        warnings.warn(
+            "Deprecated since version 0.14.0. Use .infer.",
+            DeprecationWarning,
+            stacklevel=2
+        )
+        return self.infer()
 
 
 class Definition(BaseDefinition):
@@ -513,6 +523,7 @@ class Definition(BaseDefinition):
 
         Example:
 
+        >>> from jedi._compatibility import no_unicode_pprint
         >>> from jedi import Script
         >>> source = '''
         ... def f():
@@ -525,8 +536,9 @@ class Definition(BaseDefinition):
         >>> script = Script(source, column=3)  # line is maximum by default
         >>> defs = script.goto_definitions()
         >>> defs = sorted(defs, key=lambda d: d.line)
-        >>> defs
-        [<Definition def f>, <Definition class C>]
+        >>> no_unicode_pprint(defs)  # doctest: +NORMALIZE_WHITESPACE
+        [<Definition full_name='__main__.f', description='def f'>,
+         <Definition full_name='__main__.C', description='class C'>]
         >>> str(defs[0].description)  # strip literals in python2
         'def f'
         >>> str(defs[1].description)
@@ -535,22 +547,22 @@ class Definition(BaseDefinition):
         """
         typ = self.type
         tree_name = self._name.tree_name
+        if typ == 'param':
+            return typ + ' ' + self._name.to_string()
         if typ in ('function', 'class', 'module', 'instance') or tree_name is None:
             if typ == 'function':
                 # For the description we want a short and a pythonic way.
                 typ = 'def'
-            return typ + ' ' + u(self._name.string_name)
-        elif typ == 'param':
-            return typ + ' ' + tree_name.get_definition().get_description()
+            return typ + ' ' + self._name.string_name
 
-        definition = tree_name.get_definition()
+        definition = tree_name.get_definition() or tree_name
         # Remove the prefix, because that's not what we want for get_code
         # here.
         txt = definition.get_code(include_prefix=False)
         # Delete comments:
-        txt = re.sub('#[^\n]+\n', ' ', txt)
+        txt = re.sub(r'#[^\n]+\n', ' ', txt)
         # Delete multi spaces/newlines
-        txt = re.sub('\s+', ' ', txt).strip()
+        txt = re.sub(r'\s+', ' ', txt).strip()
         return txt
 
     @property
@@ -564,7 +576,7 @@ class Definition(BaseDefinition):
         .. todo:: Add full path. This function is should return a
             `module.class.function` path.
         """
-        position = '' if self.in_builtin_module else '@%s' % (self.line)
+        position = '' if self.in_builtin_module else '@%s' % self.line
         return "%s:%s%s" % (self.module_name, self.description, position)
 
     @memoize_method
@@ -576,7 +588,7 @@ class Definition(BaseDefinition):
         """
         defs = self._name.infer()
         return sorted(
-            common.unite(defined_names(self._evaluator, d) for d in defs),
+            unite(defined_names(self._evaluator, d) for d in defs),
             key=lambda s: s._name.start_pos or (0, 0)
         )
 
@@ -603,17 +615,39 @@ class Definition(BaseDefinition):
         return hash((self._name.start_pos, self.module_path, self.name, self._evaluator))
 
 
-class CallSignature(Definition):
+class Signature(Definition):
     """
-    `CallSignature` objects is the return value of `Script.function_definition`.
+    `Signature` objects is the return value of `Script.function_definition`.
     It knows what functions you are currently in. e.g. `isinstance(` would
     return the `isinstance` function. without `(` it would return nothing.
     """
-    def __init__(self, evaluator, executable_name, bracket_start_pos, index, key_name_str):
-        super(CallSignature, self).__init__(evaluator, executable_name)
-        self._index = index
-        self._key_name_str = key_name_str
-        self._bracket_start_pos = bracket_start_pos
+    def __init__(self, evaluator, signature):
+        super(Signature, self).__init__(evaluator, signature.name)
+        self._signature = signature
+
+    @property
+    def params(self):
+        """
+        :return list of ParamDefinition:
+        """
+        return [ParamDefinition(self._evaluator, n)
+                for n in self._signature.get_param_names(resolve_stars=True)]
+
+    def to_string(self):
+        return self._signature.to_string()
+
+
+class CallSignature(Signature):
+    """
+    `CallSignature` objects is the return value of `Script.call_signatures`.
+    It knows what functions you are currently in. e.g. `isinstance(` would
+    return the `isinstance` function with its params. Without `(` it would
+    return nothing.
+    """
+    def __init__(self, evaluator, signature, call_details):
+        super(CallSignature, self).__init__(evaluator, signature)
+        self._call_details = call_details
+        self._signature = signature
 
     @property
     def index(self):
@@ -621,75 +655,65 @@ class CallSignature(Definition):
         The Param index of the current call.
         Returns None if the index cannot be found in the curent call.
         """
-        if self._key_name_str is not None:
-            for i, param in enumerate(self.params):
-                if self._key_name_str == param.name:
-                    return i
-            if self.params:
-                param_name = self.params[-1]._name
-                if param_name.tree_name is not None:
-                    if param_name.tree_name.get_definition().stars == 2:
-                        return i
-            return None
-
-        if self._index >= len(self.params):
-            for i, param in enumerate(self.params):
-                tree_name = param._name.tree_name
-                if tree_name is not None:
-                    # *args case
-                    if tree_name.get_definition().stars == 1:
-                        return i
-            return None
-        return self._index
+        return self._call_details.calculate_index(
+            self._signature.get_param_names(resolve_stars=True)
+        )
 
     @property
     def bracket_start(self):
         """
-        The indent of the bracket that is responsible for the last function
-        call.
+        The line/column of the bracket that is responsible for the last
+        function call.
         """
-        return self._bracket_start_pos
-
-    @property
-    def call_name(self):
-        """
-        .. deprecated:: 0.8.0
-           Use :attr:`.name` instead.
-        .. todo:: Remove!
-
-        The name (e.g. 'isinstance') as a string.
-        """
-        warnings.warn("Use name instead.", DeprecationWarning)
-        return self.name
-
-    @property
-    def module(self):
-        """
-        .. deprecated:: 0.8.0
-           Use :attr:`.module_name` for the module name.
-        .. todo:: Remove!
-        """
-        return self._executable.get_parent_until()
+        return self._call_details.bracket_leaf.start_pos
 
     def __repr__(self):
-        return '<%s: %s index %s>' % \
-            (type(self).__name__, self._name.string_name, self.index)
+        return '<%s: index=%r %s>' % (
+            type(self).__name__,
+            self.index,
+            self._signature.to_string(),
+        )
 
 
-class _Param(Definition):
-    """
-    Just here for backwards compatibility.
-    """
-    def get_code(self):
+class ParamDefinition(Definition):
+    def infer_default(self):
         """
-        .. deprecated:: 0.8.0
-           Use :attr:`.description` and :attr:`.name` instead.
-        .. todo:: Remove!
-
-        A function to get the whole code of the param.
+        :return list of Definition:
         """
-        warnings.warn("Use description instead.", DeprecationWarning)
-        return self.description
+        return _contexts_to_definitions(self._name.infer_default())
+
+    def infer_annotation(self, **kwargs):
+        """
+        :return list of Definition:
+
+        :param execute_annotation: If False, the values are not executed and
+            you get classes instead of instances.
+        """
+        return _contexts_to_definitions(self._name.infer_annotation(**kwargs))
+
+    def to_string(self):
+        return self._name.to_string()
+
+    @property
+    def kind(self):
+        """
+        Returns an enum instance. Returns the same values as the builtin
+        :py:attr:`inspect.Parameter.kind`.
+
+        No support for Python < 3.4 anymore.
+        """
+        if sys.version_info < (3, 5):
+            raise NotImplementedError(
+                'Python 2 is end-of-life, the new feature is not available for it'
+            )
+        return self._name.get_kind()
+
+
+def _format_signatures(context):
+    return '\n'.join(
+        signature.to_string()
+        for signature in context.get_signatures()
+    )
 
 
 class _Help(object):
@@ -701,35 +725,44 @@ class _Help(object):
         self._name = definition
 
     @memoize_method
-    def _get_node(self, fast):
-        if self._name.api_type == 'module' and not fast:
-            followed = self._name.infer()
-            if followed:
-                # TODO: Use all of the followed objects as input to Documentation.
-                context = next(iter(followed))
-                return context.tree_node
-        if self._name.tree_name is None:
-            return None
-        return self._name.tree_name.get_definition()
+    def _get_contexts(self, fast):
+        if isinstance(self._name, ImportName) and fast:
+            return {}
 
-    def full(self, fast=True):
-        node = self._get_node(fast)
-        try:
-            return node.doc
-        except AttributeError:
-            return self.raw(fast)
+        if self._name.api_type == 'statement':
+            return {}
 
-    def raw(self, fast=True):
+        return self._name.infer()
+
+    def docstring(self, fast=True, raw=True):
         """
-        The raw docstring ``__doc__`` for any object.
+        The docstring ``__doc__`` for any object.
 
         See :attr:`doc` for example.
         """
-        node = self._get_node(fast)
-        if node is None:
-            return ''
+        full_doc = ''
+        # Using the first docstring that we see.
+        for context in self._get_contexts(fast=fast):
+            if full_doc:
+                # In case we have multiple contexts, just return all of them
+                # separated by a few dashes.
+                full_doc += '\n' + '-' * 30 + '\n'
 
-        try:
-            return node.raw_doc
-        except AttributeError:
-            return ''
+            doc = context.py__doc__()
+
+            signature_text = ''
+            if self._name.is_context_name:
+                if not raw:
+                    signature_text = _format_signatures(context)
+                if not doc and context.is_stub():
+                    for c in convert_contexts(ContextSet({context}), ignore_compiled=False):
+                        doc = c.py__doc__()
+                        if doc:
+                            break
+
+            if signature_text and doc:
+                full_doc += signature_text + '\n\n' + doc
+            else:
+                full_doc += signature_text + doc
+
+        return full_doc

jedi/api/completion.py

@@ -1,13 +1,21 @@
-from jedi.parser import token
-from jedi.parser import tree
+import re
+
+from parso.python.token import PythonTokenTypes
+from parso.python import tree
+from parso.tree import search_ancestor, Leaf
+
+from jedi._compatibility import Parameter
 from jedi import debug
 from jedi import settings
 from jedi.api import classes
 from jedi.api import helpers
-from jedi.evaluate import imports
 from jedi.api import keywords
-from jedi.evaluate.helpers import evaluate_call_of_leaf
+from jedi.api.file_name import file_name_completions
+from jedi.evaluate import imports
+from jedi.evaluate.helpers import evaluate_call_of_leaf, parse_dotted_names
 from jedi.evaluate.filters import get_global_filters
+from jedi.evaluate.gradual.conversion import convert_contexts
+from jedi.parser_utils import get_statement_of_position, cut_value_at_position
 
 
 def get_call_signature_param_names(call_signatures):
@@ -15,24 +23,21 @@ def get_call_signature_param_names(call_signatures):
     for call_sig in call_signatures:
         for p in call_sig.params:
             # Allow protected access, because it's a public API.
-            tree_name = p._name.tree_name
-            # Compiled modules typically don't allow keyword arguments.
-            if tree_name is not None:
-                # Allow access on _definition here, because it's a
-                # public API and we don't want to make the internal
-                # Name object public.
-                tree_param = tree.search_ancestor(tree_name, 'param')
-                if tree_param.stars == 0:  # no *args/**kwargs
-                    yield p._name
+            if p._name.get_kind() in (Parameter.POSITIONAL_OR_KEYWORD,
+                                      Parameter.KEYWORD_ONLY):
+                yield p._name
 
 
 def filter_names(evaluator, completion_names, stack, like_name):
     comp_dct = {}
+    if settings.case_insensitive_completion:
+        like_name = like_name.lower()
     for name in completion_names:
-        if settings.case_insensitive_completion \
-                and name.string_name.lower().startswith(like_name.lower()) \
-                or name.string_name.startswith(like_name):
+        string = name.string_name
+        if settings.case_insensitive_completion:
+            string = string.lower()
 
+        if string.startswith(like_name):
             new = classes.Completion(
                 evaluator,
                 name,
@@ -51,12 +56,13 @@ def get_user_scope(module_context, position):
     """
     Returns the scope in which the user resides. This includes flows.
     """
-    user_stmt = module_context.tree_node.get_statement_for_position(position)
+    user_stmt = get_statement_of_position(module_context.tree_node, position)
     if user_stmt is None:
         def scan(scope):
             for s in scope.children:
                 if s.start_pos <= position <= s.end_pos:
-                    if isinstance(s, (tree.Scope, tree.Flow)):
+                    if isinstance(s, (tree.Scope, tree.Flow)) \
+                            or s.type in ('async_stmt', 'async_funcdef'):
                         return scan(s) or s
                     elif s.type in ('suite', 'decorated'):
                         return scan(s)
@@ -79,7 +85,7 @@ def get_flow_scope_node(module_node, position):
 
 
 class Completion:
-    def __init__(self, evaluator, module, code_lines, position, call_signatures_method):
+    def __init__(self, evaluator, module, code_lines, position, call_signatures_callback):
         self._evaluator = evaluator
         self._module_context = module
         self._module_node = module.tree_node
@@ -89,11 +95,23 @@ class Completion:
         self._like_name = helpers.get_on_completion_name(self._module_node, code_lines, position)
         # The actual cursor position is not what we need to calculate
         # everything. We want the start of the name we're on.
+        self._original_position = position
         self._position = position[0], position[1] - len(self._like_name)
-        self._call_signatures_method = call_signatures_method
+        self._call_signatures_callback = call_signatures_callback
 
     def completions(self):
-        completion_names = self._get_context_completions()
+        leaf = self._module_node.get_leaf_for_position(self._position, include_prefixes=True)
+        string, start_leaf = _extract_string_while_in_string(leaf, self._position)
+        if string is not None:
+            completions = list(file_name_completions(
+                self._evaluator, self._module_context, start_leaf, string,
+                self._like_name, self._call_signatures_callback,
+                self._code_lines, self._original_position
+            ))
+            if completions:
+                return completions
+
+        completion_names = self._get_context_completions(leaf)
 
         completions = filter_names(self._evaluator, completion_names,
                                    self.stack, self._like_name)
@@ -102,7 +120,7 @@ class Completion:
                                                   x.name.startswith('_'),
                                                   x.name.lower()))
 
-    def _get_context_completions(self):
+    def _get_context_completions(self, leaf):
         """
         Analyzes the context that a completion is made in and decides what to
         return.
@@ -118,68 +136,105 @@ class Completion:
         """
 
         grammar = self._evaluator.grammar
+        self.stack = stack = None
 
         try:
-            self.stack = helpers.get_stack_at_position(
-                grammar, self._code_lines, self._module_node, self._position
+            self.stack = stack = helpers.get_stack_at_position(
+                grammar, self._code_lines, leaf, self._position
             )
         except helpers.OnErrorLeaf as e:
-            self.stack = None
-            if e.error_leaf.value == '.':
+            value = e.error_leaf.value
+            if value == '.':
                 # After ErrorLeaf's that are dots, we will not do any
                 # completions since this probably just confuses the user.
                 return []
-            # If we don't have a context, just use global completion.
 
+            # If we don't have a context, just use global completion.
             return self._global_completions()
 
-        allowed_keywords, allowed_tokens = \
-            helpers.get_possible_completion_types(grammar, self.stack)
+        allowed_transitions = \
+            list(stack._allowed_transition_names_and_token_types())
 
-        completion_names = list(self._get_keyword_completion_names(allowed_keywords))
+        if 'if' in allowed_transitions:
+            leaf = self._module_node.get_leaf_for_position(self._position, include_prefixes=True)
+            previous_leaf = leaf.get_previous_leaf()
 
-        if token.NAME in allowed_tokens or token.INDENT in allowed_tokens:
+            indent = self._position[1]
+            if not (leaf.start_pos <= self._position <= leaf.end_pos):
+                indent = leaf.start_pos[1]
+
+            if previous_leaf is not None:
+                stmt = previous_leaf
+                while True:
+                    stmt = search_ancestor(
+                        stmt, 'if_stmt', 'for_stmt', 'while_stmt', 'try_stmt',
+                        'error_node',
+                    )
+                    if stmt is None:
+                        break
+
+                    type_ = stmt.type
+                    if type_ == 'error_node':
+                        first = stmt.children[0]
+                        if isinstance(first, Leaf):
+                            type_ = first.value + '_stmt'
+                    # Compare indents
+                    if stmt.start_pos[1] == indent:
+                        if type_ == 'if_stmt':
+                            allowed_transitions += ['elif', 'else']
+                        elif type_ == 'try_stmt':
+                            allowed_transitions += ['except', 'finally', 'else']
+                        elif type_ == 'for_stmt':
+                            allowed_transitions.append('else')
+
+        completion_names = []
+        current_line = self._code_lines[self._position[0] - 1][:self._position[1]]
+        if not current_line or current_line[-1] in ' \t.;':
+            completion_names += self._get_keyword_completion_names(allowed_transitions)
+
+        if any(t in allowed_transitions for t in (PythonTokenTypes.NAME,
+                                                  PythonTokenTypes.INDENT)):
             # This means that we actually have to do type inference.
 
-            symbol_names = list(self.stack.get_node_names(grammar))
+            nonterminals = [stack_node.nonterminal for stack_node in stack]
 
-            nodes = list(self.stack.get_nodes())
-
-            if "import_stmt" in symbol_names:
-                level = 0
-                only_modules = True
-                level, names = self._parse_dotted_names(nodes)
-                if "import_from" in symbol_names:
-                    if 'import' in nodes:
-                        only_modules = False
+            nodes = []
+            for stack_node in stack:
+                if stack_node.dfa.from_rule == 'small_stmt':
+                    nodes = []
                 else:
-                    assert "import_name" in symbol_names
+                    nodes += stack_node.nodes
 
-                completion_names += self._get_importer_names(
-                    names,
-                    level,
-                    only_modules
-                )
-            elif nodes and nodes[-1] in ('as', 'def', 'class'):
+            if nodes and nodes[-1] in ('as', 'def', 'class'):
                 # No completions for ``with x as foo`` and ``import x as foo``.
                 # Also true for defining names as a class or function.
                 return list(self._get_class_context_completions(is_function=True))
-            elif symbol_names[-1] == 'trailer' and nodes[-1] == '.':
+            elif "import_stmt" in nonterminals:
+                level, names = parse_dotted_names(nodes, "import_from" in nonterminals)
+
+                only_modules = not ("import_from" in nonterminals and 'import' in nodes)
+                completion_names += self._get_importer_names(
+                    names,
+                    level,
+                    only_modules=only_modules,
+                )
+            elif nonterminals[-1] in ('trailer', 'dotted_name') and nodes[-1] == '.':
                 dot = self._module_node.get_leaf_for_position(self._position)
                 completion_names += self._trailer_completions(dot.get_previous_leaf())
             else:
                 completion_names += self._global_completions()
                 completion_names += self._get_class_context_completions(is_function=False)
 
-            if 'trailer' in symbol_names:
-                call_signatures = self._call_signatures_method()
+            if 'trailer' in nonterminals:
+                call_signatures = self._call_signatures_callback()
                 completion_names += get_call_signature_param_names(call_signatures)
 
         return completion_names
 
-    def _get_keyword_completion_names(self, keywords_):
-        for k in keywords_:
-            yield keywords.keyword(self._evaluator, k).name
+    def _get_keyword_completion_names(self, allowed_transitions):
+        for k in allowed_transitions:
+            if isinstance(k, str) and k.isalpha():
+                yield keywords.KeywordName(self._evaluator, k)
 
     def _global_completions(self):
         context = get_user_scope(self._module_context, self._position)
@@ -203,27 +258,21 @@ class Completion:
         )
         contexts = evaluate_call_of_leaf(evaluation_context, previous_leaf)
         completion_names = []
-        debug.dbg('trailer completion contexts: %s', contexts)
+        debug.dbg('trailer completion contexts: %s', contexts, color='MAGENTA')
         for context in contexts:
             for filter in context.get_filters(
-                    search_global=False, origin_scope=user_context.tree_node):
+                    search_global=False,
+                    origin_scope=user_context.tree_node):
                 completion_names += filter.values()
-        return completion_names
 
-    def _parse_dotted_names(self, nodes):
-        level = 0
-        names = []
-        for node in nodes[1:]:
-            if node in ('.', '...'):
-                if not names:
-                    level += len(node.value)
-            elif node.type == 'dotted_name':
-                names += node.children[::2]
-            elif node.type == 'name':
-                names.append(node)
-            else:
-                break
-        return level, names
+        python_contexts = convert_contexts(contexts)
+        for c in python_contexts:
+            if c not in contexts:
+                for filter in c.get_filters(
+                        search_global=False,
+                        origin_scope=user_context.tree_node):
+                    completion_names += filter.values()
+        return completion_names
 
     def _get_importer_names(self, names, level=0, only_modules=True):
         names = [n.value for n in names]
@@ -235,7 +284,7 @@ class Completion:
         Autocomplete inherited methods when overriding in child class.
         """
         leaf = self._module_node.get_leaf_for_position(self._position, include_prefixes=True)
-        cls = leaf.get_parent_until(tree.Class)
+        cls = tree.search_ancestor(leaf, 'classdef')
         if isinstance(cls, (tree.Class, tree.Function)):
             # Complete the methods that are defined in the super classes.
             random_context = self._module_context.create_context(
@@ -253,5 +302,25 @@ class Completion:
         next(filters)
         for filter in filters:
             for name in filter.values():
+                # TODO we should probably check here for properties
                 if (name.api_type == 'function') == is_function:
                     yield name
+
+
+def _extract_string_while_in_string(leaf, position):
+    if leaf.type == 'string':
+        match = re.match(r'^\w*(\'{3}|"{3}|\'|")', leaf.value)
+        quote = match.group(1)
+        if leaf.line == position[0] and position[1] < leaf.column + match.end():
+            return None, None
+        if leaf.end_pos[0] == position[0] and position[1] > leaf.end_pos[1] - len(quote):
+            return None, None
+        return cut_value_at_position(leaf, position)[match.end():], leaf
+
+    leaves = []
+    while leaf is not None and leaf.line == position[0]:
+        if leaf.type == 'error_leaf' and ('"' in leaf.value or "'" in leaf.value):
+            return ''.join(l.get_code() for l in leaves), leaf
+        leaves.insert(0, leaf)
+        leaf = leaf.get_previous_leaf()
+    return None, None

jedi/api/environment.py

@@ -0,0 +1,458 @@
+"""
+Environments are a way to activate different Python versions or Virtualenvs for
+static analysis. The Python binary in that environment is going to be executed.
+"""
+import os
+import sys
+import hashlib
+import filecmp
+from collections import namedtuple
+
+from jedi._compatibility import highest_pickle_protocol, which
+from jedi.cache import memoize_method, time_cache
+from jedi.evaluate.compiled.subprocess import CompiledSubprocess, \
+    EvaluatorSameProcess, EvaluatorSubprocess
+
+import parso
+
+_VersionInfo = namedtuple('VersionInfo', 'major minor micro')
+
+_SUPPORTED_PYTHONS = ['3.8', '3.7', '3.6', '3.5', '3.4', '2.7']
+_SAFE_PATHS = ['/usr/bin', '/usr/local/bin']
+_CURRENT_VERSION = '%s.%s' % (sys.version_info.major, sys.version_info.minor)
+
+
+class InvalidPythonEnvironment(Exception):
+    """
+    If you see this exception, the Python executable or Virtualenv you have
+    been trying to use is probably not a correct Python version.
+    """
+
+
+class _BaseEnvironment(object):
+    @memoize_method
+    def get_grammar(self):
+        version_string = '%s.%s' % (self.version_info.major, self.version_info.minor)
+        return parso.load_grammar(version=version_string)
+
+    @property
+    def _sha256(self):
+        try:
+            return self._hash
+        except AttributeError:
+            self._hash = _calculate_sha256_for_file(self.executable)
+            return self._hash
+
+
+def _get_info():
+    return (
+        sys.executable,
+        sys.prefix,
+        sys.version_info[:3],
+    )
+
+
+class Environment(_BaseEnvironment):
+    """
+    This class is supposed to be created by internal Jedi architecture. You
+    should not create it directly. Please use create_environment or the other
+    functions instead. It is then returned by that function.
+    """
+    _subprocess = None
+
+    def __init__(self, executable):
+        self._start_executable = executable
+        # Initialize the environment
+        self._get_subprocess()
+
+    def _get_subprocess(self):
+        if self._subprocess is not None and not self._subprocess.is_crashed:
+            return self._subprocess
+
+        try:
+            self._subprocess = CompiledSubprocess(self._start_executable)
+            info = self._subprocess._send(None, _get_info)
+        except Exception as exc:
+            raise InvalidPythonEnvironment(
+                "Could not get version information for %r: %r" % (
+                    self._start_executable,
+                    exc))
+
+        # Since it could change and might not be the same(?) as the one given,
+        # set it here.
+        self.executable = info[0]
+        """
+        The Python executable, matches ``sys.executable``.
+        """
+        self.path = info[1]
+        """
+        The path to an environment, matches ``sys.prefix``.
+        """
+        self.version_info = _VersionInfo(*info[2])
+        """
+        Like ``sys.version_info``. A tuple to show the current Environment's
+        Python version.
+        """
+
+        # py2 sends bytes via pickle apparently?!
+        if self.version_info.major == 2:
+            self.executable = self.executable.decode()
+            self.path = self.path.decode()
+
+        # Adjust pickle protocol according to host and client version.
+        self._subprocess._pickle_protocol = highest_pickle_protocol([
+            sys.version_info, self.version_info])
+
+        return self._subprocess
+
+    def __repr__(self):
+        version = '.'.join(str(i) for i in self.version_info)
+        return '<%s: %s in %s>' % (self.__class__.__name__, version, self.path)
+
+    def get_evaluator_subprocess(self, evaluator):
+        return EvaluatorSubprocess(evaluator, self._get_subprocess())
+
+    @memoize_method
+    def get_sys_path(self):
+        """
+        The sys path for this environment. Does not include potential
+        modifications like ``sys.path.append``.
+
+        :returns: list of str
+        """
+        # It's pretty much impossible to generate the sys path without actually
+        # executing Python. The sys path (when starting with -S) itself depends
+        # on how the Python version was compiled (ENV variables).
+        # If you omit -S when starting Python (normal case), additionally
+        # site.py gets executed.
+        return self._get_subprocess().get_sys_path()
+
+
+class _SameEnvironmentMixin(object):
+    def __init__(self):
+        self._start_executable = self.executable = sys.executable
+        self.path = sys.prefix
+        self.version_info = _VersionInfo(*sys.version_info[:3])
+
+
+class SameEnvironment(_SameEnvironmentMixin, Environment):
+    pass
+
+
+class InterpreterEnvironment(_SameEnvironmentMixin, _BaseEnvironment):
+    def get_evaluator_subprocess(self, evaluator):
+        return EvaluatorSameProcess(evaluator)
+
+    def get_sys_path(self):
+        return sys.path
+
+
+def _get_virtual_env_from_var():
+    """Get virtualenv environment from VIRTUAL_ENV environment variable.
+
+    It uses `safe=False` with ``create_environment``, because the environment
+    variable is considered to be safe / controlled by the user solely.
+    """
+    var = os.environ.get('VIRTUAL_ENV')
+    if var:
+        # Under macOS in some cases - notably when using Pipenv - the
+        # sys.prefix of the virtualenv is /path/to/env/bin/.. instead of
+        # /path/to/env so we need to fully resolve the paths in order to
+        # compare them.
+        if os.path.realpath(var) == os.path.realpath(sys.prefix):
+            return _try_get_same_env()
+
+        try:
+            return create_environment(var, safe=False)
+        except InvalidPythonEnvironment:
+            pass
+
+
+def _calculate_sha256_for_file(path):
+    sha256 = hashlib.sha256()
+    with open(path, 'rb') as f:
+        for block in iter(lambda: f.read(filecmp.BUFSIZE), b''):
+            sha256.update(block)
+    return sha256.hexdigest()
+
+
+def get_default_environment():
+    """
+    Tries to return an active Virtualenv. If there is no VIRTUAL_ENV variable
+    set it will return the latest Python version installed on the system. This
+    makes it possible to use as many new Python features as possible when using
+    autocompletion and other functionality.
+
+    :returns: :class:`Environment`
+    """
+    virtual_env = _get_virtual_env_from_var()
+    if virtual_env is not None:
+        return virtual_env
+
+    return _try_get_same_env()
+
+
+def _try_get_same_env():
+    env = SameEnvironment()
+    if not os.path.basename(env.executable).lower().startswith('python'):
+        # This tries to counter issues with embedding. In some cases (e.g.
+        # VIM's Python Mac/Windows, sys.executable is /foo/bar/vim. This
+        # happens, because for Mac a function called `_NSGetExecutablePath` is
+        # used and for Windows `GetModuleFileNameW`. These are both platform
+        # specific functions. For all other systems sys.executable should be
+        # alright. However here we try to generalize:
+        #
+        # 1. Check if the executable looks like python (heuristic)
+        # 2. In case it's not try to find the executable
+        # 3. In case we don't find it use an interpreter environment.
+        #
+        # The last option will always work, but leads to potential crashes of
+        # Jedi - which is ok, because it happens very rarely and even less,
+        # because the code below should work for most cases.
+        if os.name == 'nt':
+            # The first case would be a virtualenv and the second a normal
+            # Python installation.
+            checks = (r'Scripts\python.exe', 'python.exe')
+        else:
+            # For unix it looks like Python is always in a bin folder.
+            checks = (
+                'bin/python%s.%s' % (sys.version_info[0], sys.version[1]),
+                'bin/python%s' % (sys.version_info[0]),
+                'bin/python',
+            )
+        for check in checks:
+            guess = os.path.join(sys.exec_prefix, check)
+            if os.path.isfile(guess):
+                # Bingo - We think we have our Python.
+                return Environment(guess)
+        # It looks like there is no reasonable Python to be found.
+        return InterpreterEnvironment()
+    # If no virtualenv is found, use the environment we're already
+    # using.
+    return env
+
+
+def get_cached_default_environment():
+    var = os.environ.get('VIRTUAL_ENV')
+    environment = _get_cached_default_environment()
+
+    # Under macOS in some cases - notably when using Pipenv - the
+    # sys.prefix of the virtualenv is /path/to/env/bin/.. instead of
+    # /path/to/env so we need to fully resolve the paths in order to
+    # compare them.
+    if var and os.path.realpath(var) != os.path.realpath(environment.path):
+        _get_cached_default_environment.clear_cache()
+        return _get_cached_default_environment()
+    return environment
+
+
+@time_cache(seconds=10 * 60)  # 10 Minutes
+def _get_cached_default_environment():
+    return get_default_environment()
+
+
+def find_virtualenvs(paths=None, **kwargs):
+    """
+    :param paths: A list of paths in your file system to be scanned for
+        Virtualenvs. It will search in these paths and potentially execute the
+        Python binaries. Also the VIRTUAL_ENV variable will be checked if it
+        contains a valid Virtualenv.
+    :param safe: Default True. In case this is False, it will allow this
+        function to execute potential `python` environments. An attacker might
+        be able to drop an executable in a path this function is searching by
+        default. If the executable has not been installed by root, it will not
+        be executed.
+
+    :yields: :class:`Environment`
+    """
+    def py27_comp(paths=None, safe=True):
+        if paths is None:
+            paths = []
+
+        _used_paths = set()
+
+        # Using this variable should be safe, because attackers might be able
+        # to drop files (via git) but not environment variables.
+        virtual_env = _get_virtual_env_from_var()
+        if virtual_env is not None:
+            yield virtual_env
+            _used_paths.add(virtual_env.path)
+
+        for directory in paths:
+            if not os.path.isdir(directory):
+                continue
+
+            directory = os.path.abspath(directory)
+            for path in os.listdir(directory):
+                path = os.path.join(directory, path)
+                if path in _used_paths:
+                    # A path shouldn't be evaluated twice.
+                    continue
+                _used_paths.add(path)
+
+                try:
+                    executable = _get_executable_path(path, safe=safe)
+                    yield Environment(executable)
+                except InvalidPythonEnvironment:
+                    pass
+
+    return py27_comp(paths, **kwargs)
+
+
+def find_system_environments():
+    """
+    Ignores virtualenvs and returns the Python versions that were installed on
+    your system. This might return nothing, if you're running Python e.g. from
+    a portable version.
+
+    The environments are sorted from latest to oldest Python version.
+
+    :yields: :class:`Environment`
+    """
+    for version_string in _SUPPORTED_PYTHONS:
+        try:
+            yield get_system_environment(version_string)
+        except InvalidPythonEnvironment:
+            pass
+
+
+# TODO: this function should probably return a list of environments since
+# multiple Python installations can be found on a system for the same version.
+def get_system_environment(version):
+    """
+    Return the first Python environment found for a string of the form 'X.Y'
+    where X and Y are the major and minor versions of Python.
+
+    :raises: :exc:`.InvalidPythonEnvironment`
+    :returns: :class:`Environment`
+    """
+    exe = which('python' + version)
+    if exe:
+        if exe == sys.executable:
+            return SameEnvironment()
+        return Environment(exe)
+
+    if os.name == 'nt':
+        for exe in _get_executables_from_windows_registry(version):
+            try:
+                return Environment(exe)
+            except InvalidPythonEnvironment:
+                pass
+    raise InvalidPythonEnvironment("Cannot find executable python%s." % version)
+
+
+def create_environment(path, safe=True):
+    """
+    Make it possible to manually create an Environment object by specifying a
+    Virtualenv path or an executable path.
+
+    :raises: :exc:`.InvalidPythonEnvironment`
+    :returns: :class:`Environment`
+    """
+    if os.path.isfile(path):
+        _assert_safe(path, safe)
+        return Environment(path)
+    return Environment(_get_executable_path(path, safe=safe))
+
+
+def _get_executable_path(path, safe=True):
+    """
+    Returns None if it's not actually a virtual env.
+    """
+
+    if os.name == 'nt':
+        python = os.path.join(path, 'Scripts', 'python.exe')
+    else:
+        python = os.path.join(path, 'bin', 'python')
+    if not os.path.exists(python):
+        raise InvalidPythonEnvironment("%s seems to be missing." % python)
+
+    _assert_safe(python, safe)
+    return python
+
+
+def _get_executables_from_windows_registry(version):
+    # The winreg module is named _winreg on Python 2.
+    try:
+        import winreg
+    except ImportError:
+        import _winreg as winreg
+
+    # TODO: support Python Anaconda.
+    sub_keys = [
+        r'SOFTWARE\Python\PythonCore\{version}\InstallPath',
+        r'SOFTWARE\Wow6432Node\Python\PythonCore\{version}\InstallPath',
+        r'SOFTWARE\Python\PythonCore\{version}-32\InstallPath',
+        r'SOFTWARE\Wow6432Node\Python\PythonCore\{version}-32\InstallPath'
+    ]
+    for root_key in [winreg.HKEY_CURRENT_USER, winreg.HKEY_LOCAL_MACHINE]:
+        for sub_key in sub_keys:
+            sub_key = sub_key.format(version=version)
+            try:
+                with winreg.OpenKey(root_key, sub_key) as key:
+                    prefix = winreg.QueryValueEx(key, '')[0]
+                    exe = os.path.join(prefix, 'python.exe')
+                    if os.path.isfile(exe):
+                        yield exe
+            except WindowsError:
+                pass
+
+
+def _assert_safe(executable_path, safe):
+    if safe and not _is_safe(executable_path):
+        raise InvalidPythonEnvironment(
+            "The python binary is potentially unsafe.")
+
+
+def _is_safe(executable_path):
+    # Resolve sym links. A venv typically is a symlink to a known Python
+    # binary. Only virtualenvs copy symlinks around.
+    real_path = os.path.realpath(executable_path)
+
+    if _is_unix_safe_simple(real_path):
+        return True
+
+    # Just check the list of known Python versions. If it's not in there,
+    # it's likely an attacker or some Python that was not properly
+    # installed in the system.
+    for environment in find_system_environments():
+        if environment.executable == real_path:
+            return True
+
+        # If the versions don't match, just compare the binary files. If we
+        # don't do that, only venvs will be working and not virtualenvs.
+        # venvs are symlinks while virtualenvs are actual copies of the
+        # Python files.
+        # This still means that if the system Python is updated and the
+        # virtualenv's Python is not (which is probably never going to get
+        # upgraded), it will not work with Jedi. IMO that's fine, because
+        # people should just be using venv. ~ dave
+        if environment._sha256 == _calculate_sha256_for_file(real_path):
+            return True
+    return False
+
+
+def _is_unix_safe_simple(real_path):
+    if _is_unix_admin():
+        # In case we are root, just be conservative and
+        # only execute known paths.
+        return any(real_path.startswith(p) for p in _SAFE_PATHS)
+
+    uid = os.stat(real_path).st_uid
+    # The interpreter needs to be owned by root. This means that it wasn't
+    # written by a user and therefore attacking Jedi is not as simple.
+    # The attack could look like the following:
+    # 1. A user clones a repository.
+    # 2. The repository has an innocent looking folder called foobar. jedi
+    #    searches for the folder and executes foobar/bin/python --version if
+    #    there's also a foobar/bin/activate.
+    # 3. The bin/python is obviously not a python script but a bash script or
+    #    whatever the attacker wants.
+    return uid == 0
+
+
+def _is_unix_admin():
+    try:
+        return os.getuid() == 0
+    except AttributeError:
+        return False  # Windows

jedi/api/exceptions.py

@@ -0,0 +1,10 @@
+class _JediError(Exception):
+    pass
+
+
+class InternalError(_JediError):
+    pass
+
+
+class WrongVersion(_JediError):
+    pass

jedi/api/file_name.py

@@ -0,0 +1,161 @@
+import os
+
+from jedi._compatibility import FileNotFoundError, force_unicode
+from jedi.evaluate.names import AbstractArbitraryName
+from jedi.api import classes
+from jedi.evaluate.helpers import get_str_or_none
+from jedi.parser_utils import get_string_quote
+
+
+def file_name_completions(evaluator, module_context, start_leaf, string,
+                          like_name, call_signatures_callback, code_lines, position):
+    # First we want to find out what can actually be changed as a name.
+    like_name_length = len(os.path.basename(string) + like_name)
+
+    addition = _get_string_additions(module_context, start_leaf)
+    if addition is None:
+        return
+    string = addition + string
+
+    # Here we use basename again, because if strings are added like
+    # `'foo' + 'bar`, it should complete to `foobar/`.
+    must_start_with = os.path.basename(string) + like_name
+    string = os.path.dirname(string)
+
+    sigs = call_signatures_callback()
+    is_in_os_path_join = sigs and all(s.full_name == 'os.path.join' for s in sigs)
+    if is_in_os_path_join:
+        to_be_added = _add_os_path_join(module_context, start_leaf, sigs[0].bracket_start)
+        if to_be_added is None:
+            is_in_os_path_join = False
+        else:
+            string = to_be_added + string
+    base_path = os.path.join(evaluator.project._path, string)
+    try:
+        listed = os.listdir(base_path)
+    except FileNotFoundError:
+        return
+    for name in listed:
+        if name.startswith(must_start_with):
+            path_for_name = os.path.join(base_path, name)
+            if is_in_os_path_join or not os.path.isdir(path_for_name):
+                if start_leaf.type == 'string':
+                    quote = get_string_quote(start_leaf)
+                else:
+                    assert start_leaf.type == 'error_leaf'
+                    quote = start_leaf.value
+                potential_other_quote = \
+                    code_lines[position[0] - 1][position[1]:position[1] + len(quote)]
+                # Add a quote if it's not already there.
+                if quote != potential_other_quote:
+                    name += quote
+            else:
+                name += os.path.sep
+
+            yield classes.Completion(
+                evaluator,
+                FileName(evaluator, name[len(must_start_with) - like_name_length:]),
+                stack=None,
+                like_name_length=like_name_length
+            )
+
+
+def _get_string_additions(module_context, start_leaf):
+    def iterate_nodes():
+        node = addition.parent
+        was_addition = True
+        for child_node in reversed(node.children[:node.children.index(addition)]):
+            if was_addition:
+                was_addition = False
+                yield child_node
+                continue
+
+            if child_node != '+':
+                break
+            was_addition = True
+
+    addition = start_leaf.get_previous_leaf()
+    if addition != '+':
+        return ''
+    context = module_context.create_context(start_leaf)
+    return _add_strings(context, reversed(list(iterate_nodes())))
+
+
+def _add_strings(context, nodes, add_slash=False):
+    string = ''
+    first = True
+    for child_node in nodes:
+        contexts = context.eval_node(child_node)
+        if len(contexts) != 1:
+            return None
+        c, = contexts
+        s = get_str_or_none(c)
+        if s is None:
+            return None
+        if not first and add_slash:
+            string += os.path.sep
+        string += force_unicode(s)
+        first = False
+    return string
+
+
+class FileName(AbstractArbitraryName):
+    api_type = u'path'
+    is_context_name = False
+
+
+def _add_os_path_join(module_context, start_leaf, bracket_start):
+    def check(maybe_bracket, nodes):
+        if maybe_bracket.start_pos != bracket_start:
+            return None
+
+        if not nodes:
+            return ''
+        context = module_context.create_context(nodes[0])
+        return _add_strings(context, nodes, add_slash=True) or ''
+
+    if start_leaf.type == 'error_leaf':
+        # Unfinished string literal, like `join('`
+        context_node = start_leaf.parent
+        index = context_node.children.index(start_leaf)
+        if index > 0:
+            error_node = context_node.children[index - 1]
+            if error_node.type == 'error_node' and len(error_node.children) >= 2:
+                index = -2
+                if error_node.children[-1].type == 'arglist':
+                    arglist_nodes = error_node.children[-1].children
+                    index -= 1
+                else:
+                    arglist_nodes = []
+
+                return check(error_node.children[index + 1], arglist_nodes[::2])
+        return None
+
+    # Maybe an arglist or some weird error case. Therefore checked below.
+    searched_node_child = start_leaf
+    while searched_node_child.parent is not None \
+            and searched_node_child.parent.type not in ('arglist', 'trailer', 'error_node'):
+        searched_node_child = searched_node_child.parent
+
+    if searched_node_child.get_first_leaf() is not start_leaf:
+        return None
+    searched_node = searched_node_child.parent
+    if searched_node is None:
+        return None
+
+    index = searched_node.children.index(searched_node_child)
+    arglist_nodes = searched_node.children[:index]
+    if searched_node.type == 'arglist':
+        trailer = searched_node.parent
+        if trailer.type == 'error_node':
+            trailer_index = trailer.children.index(searched_node)
+            assert trailer_index >= 2
+            assert trailer.children[trailer_index - 1] == '('
+            return check(trailer.children[trailer_index - 1], arglist_nodes[::2])
+        elif trailer.type == 'trailer':
+            return check(trailer.children[0], arglist_nodes[::2])
+    elif searched_node.type == 'trailer':
+        return check(searched_node.children[0], [])
+    elif searched_node.type == 'error_node':
+        # Stuff like `join(""`
+        return check(arglist_nodes[-1], [])

jedi/api/helpers.py

@@ -3,14 +3,17 @@ Helpers for the API
 """
 import re
 from collections import namedtuple
+from textwrap import dedent
 
-from jedi._compatibility import u
+from parso.python.parser import Parser
+from parso.python import tree
+
+from jedi._compatibility import u, Parameter
+from jedi.evaluate.base_context import NO_CONTEXTS
+from jedi.evaluate.syntax_tree import eval_atom
 from jedi.evaluate.helpers import evaluate_call_of_leaf
-from jedi import parser
-from jedi.parser import tree
-from jedi.parser import tokenize
-from jedi.cache import time_cache
-from jedi import common
+from jedi.evaluate.compiled import get_string_context_set
+from jedi.cache import call_signature_time_cache
 
 
 CompletionParts = namedtuple('CompletionParts', ['path', 'has_dot', 'name'])
@@ -18,7 +21,7 @@ CompletionParts = namedtuple('CompletionParts', ['path', 'has_dot', 'name'])
 
 def sorted_definitions(defs):
     # Note: `or ''` below is required because `module_path` could be
-    return sorted(defs, key=lambda x: (x.module_path or '', x.line or 0, x.column or 0))
+    return sorted(defs, key=lambda x: (x.module_path or '', x.line or 0, x.column or 0, x.name))
 
 
 def get_on_completion_name(module_node, lines, position):
@@ -42,7 +45,7 @@ def _get_code(code_lines, start_pos, end_pos):
     lines[-1] = lines[-1][:end_pos[1]]
     # Remove first line indentation.
     lines[0] = lines[0][start_pos[1]:]
-    return '\n'.join(lines)
+    return ''.join(lines)
 
 
 class OnErrorLeaf(Exception):
@@ -51,39 +54,19 @@ class OnErrorLeaf(Exception):
         return self.args[0]
 
 
-def _is_on_comment(leaf, position):
-    comment_lines = common.splitlines(leaf.prefix)
-    difference = leaf.start_pos[0] - position[0]
-    prefix_start_pos = leaf.get_start_pos_of_prefix()
-    if difference == 0:
-        indent = leaf.start_pos[1]
-    elif position[0] == prefix_start_pos[0]:
-        indent = prefix_start_pos[1]
-    else:
-        indent = 0
-    line = comment_lines[-difference - 1][:position[1] - indent]
-    return '#' in line
-
-
-def _get_code_for_stack(code_lines, module_node, position):
-    leaf = module_node.get_leaf_for_position(position, include_prefixes=True)
+def _get_code_for_stack(code_lines, leaf, position):
     # It might happen that we're on whitespace or on a comment. This means
     # that we would not get the right leaf.
     if leaf.start_pos >= position:
-        if _is_on_comment(leaf, position):
-            return u('')
-
         # If we're not on a comment simply get the previous leaf and proceed.
-        try:
-            leaf = leaf.get_previous_leaf()
-        except IndexError:
+        leaf = leaf.get_previous_leaf()
+        if leaf is None:
             return u('')  # At the beginning of the file.
 
     is_after_newline = leaf.type == 'newline'
     while leaf.type == 'newline':
-        try:
-            leaf = leaf.get_previous_leaf()
-        except IndexError:
+        leaf = leaf.get_previous_leaf()
+        if leaf is None:
             return u('')
 
     if leaf.type == 'error_leaf' or leaf.type == 'string':
@@ -95,11 +78,10 @@ def _get_code_for_stack(code_lines, module_node, position):
         # impossible.
         raise OnErrorLeaf(leaf)
     else:
-        if leaf == ';':
-            user_stmt = leaf.parent
-        else:
-            user_stmt = leaf.get_definition()
-        if user_stmt.parent.type == 'simple_stmt':
+        user_stmt = leaf
+        while True:
+            if user_stmt.parent.type in ('file_input', 'suite', 'simple_stmt'):
+                break
             user_stmt = user_stmt.parent
 
         if is_after_newline:
@@ -112,7 +94,7 @@ def _get_code_for_stack(code_lines, module_node, position):
         return _get_code(code_lines, user_stmt.get_start_pos_of_prefix(), position)
 
 
-def get_stack_at_position(grammar, code_lines, module_node, pos):
+def get_stack_at_position(grammar, code_lines, leaf, pos):
     """
     Returns the possible node names (e.g. import_from, xor_test or yield_stmt).
     """
@@ -120,77 +102,38 @@ def get_stack_at_position(grammar, code_lines, module_node, pos):
         pass
 
     def tokenize_without_endmarker(code):
-        tokens = tokenize.source_tokens(code, use_exact_op_types=True)
-        for token_ in tokens:
-            if token_.string == safeword:
+        # TODO This is for now not an official parso API that exists purely
+        #   for Jedi.
+        tokens = grammar._tokenize(code)
+        for token in tokens:
+            if token.string == safeword:
+                raise EndMarkerReached()
+            elif token.prefix.endswith(safeword):
+                # This happens with comments.
+                raise EndMarkerReached()
+            elif token.string.endswith(safeword):
+                yield token  # Probably an f-string literal that was not finished.
                 raise EndMarkerReached()
             else:
-                yield token_
+                yield token
 
-    code = _get_code_for_stack(code_lines, module_node, pos)
+    # The code might be indedented, just remove it.
+    code = dedent(_get_code_for_stack(code_lines, leaf, pos))
     # We use a word to tell Jedi when we have reached the start of the
     # completion.
     # Use Z as a prefix because it's not part of a number suffix.
     safeword = 'ZZZ_USER_WANTS_TO_COMPLETE_HERE_WITH_JEDI'
-    code = code + safeword
+    code = code + ' ' + safeword
 
-    p = parser.ParserWithRecovery(grammar, code, start_parsing=False)
+    p = Parser(grammar._pgen_grammar, error_recovery=True)
     try:
-        p.parse(tokenizer=tokenize_without_endmarker(code))
+        p.parse(tokens=tokenize_without_endmarker(code))
     except EndMarkerReached:
-        return Stack(p.pgen_parser.stack)
-    raise SystemError("This really shouldn't happen. There's a bug in Jedi.")
-
-
-class Stack(list):
-    def get_node_names(self, grammar):
-        for dfa, state, (node_number, nodes) in self:
-            yield grammar.number2symbol[node_number]
-
-    def get_nodes(self):
-        for dfa, state, (node_number, nodes) in self:
-            for node in nodes:
-                yield node
-
-
-def get_possible_completion_types(grammar, stack):
-    def add_results(label_index):
-        try:
-            grammar_labels.append(inversed_tokens[label_index])
-        except KeyError:
-            try:
-                keywords.append(inversed_keywords[label_index])
-            except KeyError:
-                t, v = grammar.labels[label_index]
-                assert t >= 256
-                # See if it's a symbol and if we're in its first set
-                inversed_keywords
-                itsdfa = grammar.dfas[t]
-                itsstates, itsfirst = itsdfa
-                for first_label_index in itsfirst.keys():
-                    add_results(first_label_index)
-
-    inversed_keywords = dict((v, k) for k, v in grammar.keywords.items())
-    inversed_tokens = dict((v, k) for k, v in grammar.tokens.items())
-
-    keywords = []
-    grammar_labels = []
-
-    def scan_stack(index):
-        dfa, state, node = stack[index]
-        states, first = dfa
-        arcs = states[state]
-
-        for label_index, new_state in arcs:
-            if label_index == 0:
-                # An accepting state, check the stack below.
-                scan_stack(index - 1)
-            else:
-                add_results(label_index)
-
-    scan_stack(-1)
-
-    return keywords, grammar_labels
+        return p.stack
+    raise SystemError(
+        "This really shouldn't happen. There's a bug in Jedi:\n%s"
+        % list(tokenize_without_endmarker(code))
+    )
 
 
 def evaluate_goto_definition(evaluator, context, leaf):
@@ -200,19 +143,154 @@ def evaluate_goto_definition(evaluator, context, leaf):
         return evaluator.goto_definitions(context, leaf)
 
     parent = leaf.parent
+    definitions = NO_CONTEXTS
     if parent.type == 'atom':
-        return context.eval_node(leaf.parent)
+        # e.g. `(a + b)`
+        definitions = context.eval_node(leaf.parent)
     elif parent.type == 'trailer':
-        return evaluate_call_of_leaf(context, leaf)
+        # e.g. `a()`
+        definitions = evaluate_call_of_leaf(context, leaf)
     elif isinstance(leaf, tree.Literal):
-        return context.evaluator.eval_atom(context, leaf)
-    return []
+        # e.g. `"foo"` or `1.0`
+        return eval_atom(context, leaf)
+    elif leaf.type in ('fstring_string', 'fstring_start', 'fstring_end'):
+        return get_string_context_set(evaluator)
+    return definitions
 
 
-CallSignatureDetails = namedtuple(
-    'CallSignatureDetails',
-    ['bracket_leaf', 'call_index', 'keyword_name_str']
-)
+class CallDetails(object):
+    def __init__(self, bracket_leaf, children, position):
+        ['bracket_leaf', 'call_index', 'keyword_name_str']
+        self.bracket_leaf = bracket_leaf
+        self._children = children
+        self._position = position
+
+    @property
+    def index(self):
+        return _get_index_and_key(self._children, self._position)[0]
+
+    @property
+    def keyword_name_str(self):
+        return _get_index_and_key(self._children, self._position)[1]
+
+    def calculate_index(self, param_names):
+        positional_count = 0
+        used_names = set()
+        star_count = -1
+        args = list(_iter_arguments(self._children, self._position))
+        if not args:
+            if param_names:
+                return 0
+            else:
+                return None
+
+        is_kwarg = False
+        for i, (star_count, key_start, had_equal) in enumerate(args):
+            is_kwarg |= had_equal | (star_count == 2)
+            if star_count:
+                pass  # For now do nothing, we don't know what's in there here.
+            else:
+                if i + 1 != len(args):  # Not last
+                    if had_equal:
+                        used_names.add(key_start)
+                    else:
+                        positional_count += 1
+
+        for i, param_name in enumerate(param_names):
+            kind = param_name.get_kind()
+
+            if not is_kwarg:
+                if kind == Parameter.VAR_POSITIONAL:
+                    return i
+                if kind in (Parameter.POSITIONAL_OR_KEYWORD, Parameter.POSITIONAL_ONLY):
+                    if i == positional_count:
+                        return i
+
+            if key_start is not None and not star_count == 1 or star_count == 2:
+                if param_name.string_name not in used_names \
+                        and (kind == Parameter.KEYWORD_ONLY
+                             or kind == Parameter.POSITIONAL_OR_KEYWORD
+                             and positional_count <= i):
+                    if star_count:
+                        return i
+                    if had_equal:
+                        if param_name.string_name == key_start:
+                            return i
+                    else:
+                        if param_name.string_name.startswith(key_start):
+                            return i
+
+                if kind == Parameter.VAR_KEYWORD:
+                    return i
+        return None
+
+
+def _iter_arguments(nodes, position):
+    def remove_after_pos(name):
+        if name.type != 'name':
+            return None
+        return name.value[:position[1] - name.start_pos[1]]
+
+    # Returns Generator[Tuple[star_count, Optional[key_start: str], had_equal]]
+    nodes_before = [c for c in nodes if c.start_pos < position]
+    if nodes_before[-1].type == 'arglist':
+        for x in _iter_arguments(nodes_before[-1].children, position):
+            yield x  # Python 2 :(
+        return
+
+    previous_node_yielded = False
+    stars_seen = 0
+    for i, node in enumerate(nodes_before):
+        if node.type == 'argument':
+            previous_node_yielded = True
+            first = node.children[0]
+            second = node.children[1]
+            if second == '=':
+                if second.start_pos < position:
+                    yield 0, first.value, True
+                else:
+                    yield 0, remove_after_pos(first), False
+            elif first in ('*', '**'):
+                yield len(first.value), remove_after_pos(second), False
+            else:
+                # Must be a Comprehension
+                first_leaf = node.get_first_leaf()
+                if first_leaf.type == 'name' and first_leaf.start_pos >= position:
+                    yield 0, remove_after_pos(first_leaf), False
+                else:
+                    yield 0, None, False
+            stars_seen = 0
+        elif node.type in ('testlist', 'testlist_star_expr'):  # testlist is Python 2
+            for n in node.children[::2]:
+                if n.type == 'star_expr':
+                    stars_seen = 1
+                    n = n.children[1]
+                yield stars_seen, remove_after_pos(n), False
+                stars_seen = 0
+            # The count of children is even if there's a comma at the end.
+            previous_node_yielded = bool(len(node.children) % 2)
+        elif isinstance(node, tree.PythonLeaf) and node.value == ',':
+            if not previous_node_yielded:
+                yield stars_seen, '', False
+                stars_seen = 0
+            previous_node_yielded = False
+        elif isinstance(node, tree.PythonLeaf) and node.value in ('*', '**'):
+            stars_seen = len(node.value)
+        elif node == '=' and nodes_before[-1]:
+            previous_node_yielded = True
+            before = nodes_before[i - 1]
+            if before.type == 'name':
+                yield 0, before.value, True
+            else:
+                yield 0, None, False
+            # Just ignore the star that is probably a syntax error.
+            stars_seen = 0
+
+    if not previous_node_yielded:
+        if nodes_before[-1].type == 'name':
+            yield stars_seen, remove_after_pos(nodes_before[-1]), False
+        else:
+            yield stars_seen, '', False
 
 
 def _get_index_and_key(nodes, position):
@@ -221,22 +299,22 @@ def _get_index_and_key(nodes, position):
     """
     nodes_before = [c for c in nodes if c.start_pos < position]
     if nodes_before[-1].type == 'arglist':
-        nodes_before = [c for c in nodes_before[-1].children if c.start_pos < position]
+        return _get_index_and_key(nodes_before[-1].children, position)
 
     key_str = None
 
-    if nodes_before:
-        last = nodes_before[-1]
-        if last.type == 'argument' and last.children[1].end_pos <= position:
-            # Checked if the argument
-            key_str = last.children[0].value
-        elif last == '=':
-            key_str = nodes_before[-2].value
+    last = nodes_before[-1]
+    if last.type == 'argument' and last.children[1] == '=' \
+            and last.children[1].end_pos <= position:
+        # Checked if the argument
+        key_str = last.children[0].value
+    elif last == '=':
+        key_str = nodes_before[-2].value
 
     return nodes_before.count(','), key_str
 
 
-def _get_call_signature_details_from_error_node(node, position):
+def _get_call_signature_details_from_error_node(node, additional_children, position):
     for index, element in reversed(list(enumerate(node.children))):
         # `index > 0` means that it's a trailer and not an atom.
         if element == '(' and element.end_pos <= position and index > 0:
@@ -244,23 +322,22 @@ def _get_call_signature_details_from_error_node(node, position):
             # until the parentheses is enough.
             children = node.children[index:]
             name = element.get_previous_leaf()
+            if name is None:
+                continue
             if name.type == 'name' or name.parent.type in ('trailer', 'atom'):
-                return CallSignatureDetails(
-                    element,
-                    *_get_index_and_key(children, position)
-                )
+                return CallDetails(element, children + additional_children, position)
 
 
 def get_call_signature_details(module, position):
     leaf = module.get_leaf_for_position(position, include_prefixes=True)
     if leaf.start_pos >= position:
         # Whitespace / comments after the leaf count towards the previous leaf.
-        try:
-            leaf = leaf.get_previous_leaf()
-        except IndexError:
+        leaf = leaf.get_previous_leaf()
+        if leaf is None:
             return None
 
     if leaf == ')':
+        # TODO is this ok?
         if leaf.end_pos == position:
             leaf = leaf.get_next_leaf()
 
@@ -273,33 +350,42 @@ def get_call_signature_details(module, position):
             # makes it feel strange to have a call signature.
             return None
 
-        for n in node.children[::-1]:
-            if n.start_pos < position and n.type == 'error_node':
-                result = _get_call_signature_details_from_error_node(n, position)
-                if result is not None:
-                    return result
+        additional_children = []
+        for n in reversed(node.children):
+            if n.start_pos < position:
+                if n.type == 'error_node':
+                    result = _get_call_signature_details_from_error_node(
+                        n, additional_children, position
+                    )
+                    if result is not None:
+                        return result
+
+                    additional_children[0:0] = n.children
+                    continue
+                additional_children.insert(0, n)
 
         if node.type == 'trailer' and node.children[0] == '(':
             leaf = node.get_previous_leaf()
-            return CallSignatureDetails(
-                node.children[0], *_get_index_and_key(node.children, position))
+            if leaf is None:
+                return None
+            return CallDetails(node.children[0], node.children, position)
 
         node = node.parent
 
     return None
 
 
-@time_cache("call_signatures_validity")
+@call_signature_time_cache("call_signatures_validity")
 def cache_call_signatures(evaluator, context, bracket_leaf, code_lines, user_pos):
     """This function calculates the cache key."""
-    index = user_pos[0] - 1
+    line_index = user_pos[0] - 1
 
-    before_cursor = code_lines[index][:user_pos[1]]
-    other_lines = code_lines[bracket_leaf.start_pos[0]:index]
-    whole = '\n'.join(other_lines + [before_cursor])
+    before_cursor = code_lines[line_index][:user_pos[1]]
+    other_lines = code_lines[bracket_leaf.start_pos[0]:line_index]
+    whole = ''.join(other_lines + [before_cursor])
     before_bracket = re.match(r'.*\(', whole, re.DOTALL)
 
-    module_path = bracket_leaf.get_parent_until().path
+    module_path = context.get_root_context().py__file__()
     if module_path is None:
         yield None  # Don't cache!
     else:
@@ -307,5 +393,5 @@ def cache_call_signatures(evaluator, context, bracket_leaf, code_lines, user_pos
     yield evaluate_goto_definition(
         evaluator,
         context,
-        bracket_leaf.get_previous_leaf()
+        bracket_leaf.get_previous_leaf(),
     )

jedi/api/interpreter.py

@@ -2,41 +2,46 @@
 TODO Some parts of this module are still not well documented.
 """
 
-from jedi.evaluate.representation import ModuleContext
+from jedi.evaluate.context import ModuleContext
 from jedi.evaluate import compiled
 from jedi.evaluate.compiled import mixed
-from jedi.evaluate.context import Context
+from jedi.evaluate.compiled.access import create_access_path
+from jedi.evaluate.base_context import ContextWrapper
 
 
-class MixedModuleContext(Context):
-    resets_positions = True
+def _create(evaluator, obj):
+    return compiled.create_from_access_path(
+        evaluator, create_access_path(evaluator, obj)
+    )
+
+
+class NamespaceObject(object):
+    def __init__(self, dct):
+        self.__dict__ = dct
+
+
+class MixedModuleContext(ContextWrapper):
     type = 'mixed_module'
 
-    def __init__(self, evaluator, tree_module, namespaces):
-        self.evaluator = evaluator
-        self._namespaces = namespaces
-
-        self._namespace_objects = [type('jedi_namespace', (), n) for n in namespaces]
-        self._module_context = ModuleContext(evaluator, tree_module)
-        self.tree_node = tree_module
-
-    def get_node(self):
-        return self.tree_node
+    def __init__(self, evaluator, tree_module, namespaces, file_io, code_lines):
+        module_context = ModuleContext(
+            evaluator, tree_module,
+            file_io=file_io,
+            string_names=('__main__',),
+            code_lines=code_lines
+        )
+        super(MixedModuleContext, self).__init__(module_context)
+        self._namespace_objects = [NamespaceObject(n) for n in namespaces]
 
     def get_filters(self, *args, **kwargs):
-        for filter in self._module_context.get_filters(*args, **kwargs):
+        for filter in self._wrapped_context.get_filters(*args, **kwargs):
             yield filter
 
         for namespace_obj in self._namespace_objects:
-            compiled_object = compiled.create(self.evaluator, namespace_obj)
+            compiled_object = _create(self.evaluator, namespace_obj)
             mixed_object = mixed.MixedObject(
-                self.evaluator,
-                parent_context=self,
                 compiled_object=compiled_object,
-                tree_name=self.tree_node.name
+                tree_context=self._wrapped_context
             )
             for filter in mixed_object.get_filters(*args, **kwargs):
                 yield filter
-
-    def __getattr__(self, name):
-        return getattr(self._module_context, name)

jedi/api/keywords.py

@@ -1,10 +1,8 @@
 import pydoc
-import keyword
 
-from jedi._compatibility import is_py3, is_py35
-from jedi import common
-from jedi.evaluate.filters import AbstractNameDefinition
-from jedi.parser.tree import Leaf
+from jedi.evaluate.utils import ignored
+from jedi.evaluate.names import AbstractArbitraryName
+
 try:
     from pydoc_data import topics as pydoc_topics
 except ImportError:
@@ -16,95 +14,39 @@ except ImportError:
         # pydoc_data module in its file python3x.zip.
         pydoc_topics = None
 
-if is_py3:
-    if is_py35:
-        # in python 3.5 async and await are not proper keywords, but for
-        # completion pursposes should as as though they are
-        keys = keyword.kwlist + ["async", "await"]
-    else:
-        keys = keyword.kwlist
-else:
-    keys = keyword.kwlist + ['None', 'False', 'True']
-
-
-def has_inappropriate_leaf_keyword(pos, module):
-    relevant_errors = filter(
-        lambda error: error.first_pos[0] == pos[0],
-        module.error_statement_stacks)
-
-    for error in relevant_errors:
-        if error.next_token in keys:
-            return True
-
-    return False
-
-
-def completion_names(evaluator, stmt, pos, module):
-    keyword_list = all_keywords(evaluator)
-
-    if not isinstance(stmt, Leaf) or has_inappropriate_leaf_keyword(pos, module):
-        keyword_list = filter(
-            lambda keyword: not keyword.only_valid_as_leaf,
-            keyword_list
-        )
-    return [keyword.name for keyword in keyword_list]
-
-
-def all_keywords(evaluator, pos=(0, 0)):
-    return set([Keyword(evaluator, k, pos) for k in keys])
-
-
-def keyword(evaluator, string, pos=(0, 0)):
-    if string in keys:
-        return Keyword(evaluator, string, pos)
-    else:
-        return None
-
 
 def get_operator(evaluator, string, pos):
     return Keyword(evaluator, string, pos)
 
 
-keywords_only_valid_as_leaf = (
-    'continue',
-    'break',
-)
+class KeywordName(AbstractArbitraryName):
+    api_type = u'keyword'
 
-
-class KeywordName(AbstractNameDefinition):
-    api_type = 'keyword'
-
-    def __init__(self, evaluator, name):
-        self.string_name = name
-        self.parent_context = evaluator.BUILTINS
-
-    def eval(self):
-        return set()
+    def infer(self):
+        return [Keyword(self.evaluator, self.string_name, (0, 0))]
 
 
 class Keyword(object):
-    api_type = 'keyword'
+    api_type = u'keyword'
 
     def __init__(self, evaluator, name, pos):
         self.name = KeywordName(evaluator, name)
         self.start_pos = pos
-        self.parent = evaluator.BUILTINS
-
-    def get_parent_until(self):
-        return self.parent
-
-    @property
-    def only_valid_as_leaf(self):
-        return self.name.value in keywords_only_valid_as_leaf
+        self.parent = evaluator.builtins_module
 
     @property
     def names(self):
         """ For a `parsing.Name` like comparision """
         return [self.name]
 
-    @property
-    def docstr(self):
-        return imitate_pydoc(self.name)
+    def py__doc__(self):
+        return imitate_pydoc(self.name.string_name)
+
+    def get_signatures(self):
+        # TODO this makes no sense, I think Keyword should somehow merge with
+        #   Context to make it easier for the api/classes.py to deal with all
+        #   of it.
+        return []
 
     def __repr__(self):
         return '<%s: %s>' % (type(self).__name__, self.name)
@@ -122,7 +64,7 @@ def imitate_pydoc(string):
     # with unicode strings)
     string = str(string)
     h = pydoc.help
-    with common.ignored(KeyError):
+    with ignored(KeyError):
         # try to access symbols
         string = h.symbols[string]
         string, _, related = string.partition(' ')
@@ -138,6 +80,6 @@ def imitate_pydoc(string):
         return ''
 
     try:
-        return pydoc_topics.topics[label] if pydoc_topics else ''
+        return pydoc_topics.topics[label].strip() if pydoc_topics else ''
     except KeyError:
         return ''

jedi/api/project.py

@@ -0,0 +1,197 @@
+import os
+import json
+
+from jedi._compatibility import FileNotFoundError, PermissionError, IsADirectoryError
+from jedi.api.environment import SameEnvironment, \
+    get_cached_default_environment
+from jedi.api.exceptions import WrongVersion
+from jedi._compatibility import force_unicode
+from jedi.evaluate.sys_path import discover_buildout_paths
+from jedi.evaluate.cache import evaluator_as_method_param_cache
+from jedi.common.utils import traverse_parents
+
+_CONFIG_FOLDER = '.jedi'
+_CONTAINS_POTENTIAL_PROJECT = 'setup.py', '.git', '.hg', 'requirements.txt', 'MANIFEST.in'
+
+_SERIALIZER_VERSION = 1
+
+
+def _remove_duplicates_from_path(path):
+    used = set()
+    for p in path:
+        if p in used:
+            continue
+        used.add(p)
+        yield p
+
+
+def _force_unicode_list(lst):
+    return list(map(force_unicode, lst))
+
+
+class Project(object):
+    # TODO serialize environment
+    _serializer_ignore_attributes = ('_environment',)
+    _environment = None
+
+    @staticmethod
+    def _get_json_path(base_path):
+        return os.path.join(base_path, _CONFIG_FOLDER, 'project.json')
+
+    @classmethod
+    def load(cls, path):
+        """
+        :param path: The path of the directory you want to use as a project.
+        """
+        with open(cls._get_json_path(path)) as f:
+            version, data = json.load(f)
+
+        if version == 1:
+            self = cls.__new__()
+            self.__dict__.update(data)
+            return self
+        else:
+            raise WrongVersion(
+                "The Jedi version of this project seems newer than what we can handle."
+            )
+
+    def __init__(self, path, **kwargs):
+        """
+        :param path: The base path for this project.
+        :param sys_path: list of str. You can override the sys path if you
+            want. By default the ``sys.path.`` is generated from the
+            environment (virtualenvs, etc).
+        :param smart_sys_path: If this is enabled (default), adds paths from
+            local directories. Otherwise you will have to rely on your packages
+            being properly configured on the ``sys.path``.
+        """
+        def py2_comp(path, environment=None, sys_path=None,
+                     smart_sys_path=True, _django=False):
+            self._path = os.path.abspath(path)
+            if isinstance(environment, SameEnvironment):
+                self._environment = environment
+
+            self._sys_path = sys_path
+            self._smart_sys_path = smart_sys_path
+            self._django = _django
+
+        py2_comp(path, **kwargs)
+
+    @evaluator_as_method_param_cache()
+    def _get_base_sys_path(self, evaluator, environment=None):
+        if self._sys_path is not None:
+            return self._sys_path
+
+        # The sys path has not been set explicitly.
+        if environment is None:
+            environment = self.get_environment()
+
+        sys_path = list(environment.get_sys_path())
+        try:
+            sys_path.remove('')
+        except ValueError:
+            pass
+        return sys_path
+
+    @evaluator_as_method_param_cache()
+    def _get_sys_path(self, evaluator, environment=None, add_parent_paths=True):
+        """
+        Keep this method private for all users of jedi. However internally this
+        one is used like a public method.
+        """
+        suffixed = []
+        prefixed = []
+
+        sys_path = list(self._get_base_sys_path(evaluator, environment))
+        if self._smart_sys_path:
+            prefixed.append(self._path)
+
+            if evaluator.script_path is not None:
+                suffixed += discover_buildout_paths(evaluator, evaluator.script_path)
+
+                if add_parent_paths:
+                    traversed = list(traverse_parents(evaluator.script_path))
+
+                    # AFAIK some libraries have imports like `foo.foo.bar`, which
+                    # leads to the conclusion to by default prefer longer paths
+                    # rather than shorter ones by default.
+                    suffixed += reversed(traversed)
+
+        if self._django:
+            prefixed.append(self._path)
+
+        path = prefixed + sys_path + suffixed
+        return list(_force_unicode_list(_remove_duplicates_from_path(path)))
+
+    def save(self):
+        data = dict(self.__dict__)
+        for attribute in self._serializer_ignore_attributes:
+            data.pop(attribute, None)
+
+        with open(self._get_json_path(self._path), 'wb') as f:
+            return json.dump((_SERIALIZER_VERSION, data), f)
+
+    def get_environment(self):
+        if self._environment is None:
+            return get_cached_default_environment()
+
+        return self._environment
+
+    def __repr__(self):
+        return '<%s: %s>' % (self.__class__.__name__, self._path)
+
+
+def _is_potential_project(path):
+    for name in _CONTAINS_POTENTIAL_PROJECT:
+        if os.path.exists(os.path.join(path, name)):
+            return True
+    return False
+
+
+def _is_django_path(directory):
+    """ Detects the path of the very well known Django library (if used) """
+    try:
+        with open(os.path.join(directory, 'manage.py'), 'rb') as f:
+            return b"DJANGO_SETTINGS_MODULE" in f.read()
+    except (FileNotFoundError, IsADirectoryError, PermissionError):
+        return False
+
+    return False
+
+
+def get_default_project(path=None):
+    if path is None:
+        path = os.getcwd()
+
+    check = os.path.realpath(path)
+    probable_path = None
+    first_no_init_file = None
+    for dir in traverse_parents(check, include_current=True):
+        try:
+            return Project.load(dir)
+        except (FileNotFoundError, IsADirectoryError, PermissionError):
+            pass
+
+        if first_no_init_file is None:
+            if os.path.exists(os.path.join(dir, '__init__.py')):
+                # In the case that a __init__.py exists, it's in 99% just a
+                # Python package and the project sits at least one level above.
+                continue
+            else:
+                first_no_init_file = dir
+
+        if _is_django_path(dir):
+            return Project(dir, _django=True)
+
+        if probable_path is None and _is_potential_project(dir):
+            probable_path = dir
+
+    if probable_path is not None:
+        # TODO search for setup.py etc
+        return Project(probable_path)
+
+    if first_no_init_file is not None:
+        return Project(first_no_init_file)
+
+    curdir = path if os.path.isdir(path) else os.path.dirname(path)
+    return Project(curdir)

jedi/api/replstartup.py

@@ -1,6 +1,8 @@
 """
 To use Jedi completion in Python interpreter, add the following in your shell
-setup (e.g., ``.bashrc``)::
+setup (e.g., ``.bashrc``). This works only on Linux/Mac, because readline is
+not available on Windows. If you still want Jedi autocompletion in your REPL,
+just use IPython instead::
 
     export PYTHONSTARTUP="$(python -m jedi repl)"
 
@@ -11,8 +13,8 @@ Then you will be able to use Jedi completer in your Python interpreter::
     [GCC 4.6.1] on linux2
     Type "help", "copyright", "credits" or "license" for more information.
     >>> import os
-    >>> os.path.join().split().in<TAB>                     # doctest: +SKIP
-    os.path.join().split().index   os.path.join().split().insert
+    >>> os.path.join('a', 'b').split().in<TAB>            # doctest: +SKIP
+    ..dex   ..sert
 
 """
 import jedi.utils

jedi/api/usages.py

@@ -1,68 +0,0 @@
-from jedi.api import classes
-from jedi.parser import tree
-from jedi.evaluate import imports
-from jedi.evaluate.filters import TreeNameDefinition
-from jedi.evaluate.representation import ModuleContext
-
-
-def usages(evaluator, definition_names, mods):
-    """
-    :param definitions: list of Name
-    """
-    def resolve_names(definition_names):
-        for name in definition_names:
-            if name.api_type == 'module':
-                found = False
-                for context in name.infer():
-                    found = True
-                    yield context.name
-                if not found:
-                    yield name
-            else:
-                yield name
-
-    def compare_array(definition_names):
-        """ `definitions` are being compared by module/start_pos, because
-        sometimes the id's of the objects change (e.g. executions).
-        """
-        return [
-            (name.get_root_context(), name.start_pos)
-            for name in resolve_names(definition_names)
-        ]
-
-    search_name = list(definition_names)[0].string_name
-    compare_definitions = compare_array(definition_names)
-    mods = mods | set([d.get_root_context() for d in definition_names])
-    definition_names = set(resolve_names(definition_names))
-    for m in imports.get_modules_containing_name(evaluator, mods, search_name):
-        if isinstance(m, ModuleContext):
-            for name_node in m.tree_node.used_names.get(search_name, []):
-                context = evaluator.create_context(m, name_node)
-                result = evaluator.goto(context, name_node)
-                if [c for c in compare_array(result) if c in compare_definitions]:
-                    name = TreeNameDefinition(context, name_node)
-                    definition_names.add(name)
-                    # Previous definitions might be imports, so include them
-                    # (because goto might return that import name).
-                    compare_definitions += compare_array([name])
-        else:
-            # compiled objects
-            definition_names.add(m.name)
-
-    return [classes.Definition(evaluator, n) for n in definition_names]
-
-
-def resolve_potential_imports(evaluator, definitions):
-    """ Adds the modules of the imports """
-    new = set()
-    for d in definitions:
-        if isinstance(d, TreeNameDefinition):
-            imp_or_stmt = d.tree_name.get_definition()
-            if isinstance(imp_or_stmt, tree.Import):
-                new |= resolve_potential_imports(
-                    evaluator,
-                    set(imports.infer_import(
-                        d.parent_context, d.tree_name, is_goto=True
-                    ))
-                )
-    return set(definitions) | new

jedi/cache.py

@@ -3,8 +3,6 @@ This caching is very important for speed and memory optimizations. There's
 nothing really spectacular, just some decorators. The following cache types are
 available:
 
-- module caching (`load_parser` and `save_parser`), which uses pickle and is
-  really important to assure low load times of modules like ``numpy``.
 - ``time_cache`` can be used to cache something for just a limited time span,
   which can be useful if there's user interaction and the user cannot react
   faster than a certain time.
@@ -14,14 +12,46 @@ there are global variables, which are holding the cache information. Some of
 these variables are being cleaned after every API usage.
 """
 import time
+from functools import wraps
 
 from jedi import settings
-from jedi.parser.utils import parser_cache
-from jedi.parser.utils import underscore_memoization
+from parso.cache import parser_cache
 
 _time_caches = {}
 
 
+def underscore_memoization(func):
+    """
+    Decorator for methods::
+
+        class A(object):
+            def x(self):
+                if self._x:
+                    self._x = 10
+                return self._x
+
+    Becomes::
+
+        class A(object):
+            @underscore_memoization
+            def x(self):
+                return 10
+
+    A now has an attribute ``_x`` written by this decorator.
+    """
+    name = '_' + func.__name__
+
+    def wrapper(self):
+        try:
+            return getattr(self, name)
+        except AttributeError:
+            result = func(self)
+            setattr(self, name, result)
+            return result
+
+    return wrapper
+
+
 def clear_time_caches(delete_all=False):
     """ Jedi caches many things, that should be completed after each completion
     finishes.
@@ -45,7 +75,7 @@ def clear_time_caches(delete_all=False):
                     del tc[key]
 
 
-def time_cache(time_add_setting):
+def call_signature_time_cache(time_add_setting):
     """
     This decorator works as follows: Call it with a setting and after that
     use the function with a callable that returns the key.
@@ -77,8 +107,32 @@ def time_cache(time_add_setting):
     return _temp
 
 
+def time_cache(seconds):
+    def decorator(func):
+        cache = {}
+
+        @wraps(func)
+        def wrapper(*args, **kwargs):
+            key = (args, frozenset(kwargs.items()))
+            try:
+                created, result = cache[key]
+                if time.time() < created + seconds:
+                    return result
+            except KeyError:
+                pass
+            result = func(*args, **kwargs)
+            cache[key] = time.time(), result
+            return result
+
+        wrapper.clear_cache = lambda: cache.clear()
+        return wrapper
+
+    return decorator
+
+
 def memoize_method(method):
     """A normal memoize function."""
+    @wraps(method)
     def wrapper(self, *args, **kwargs):
         cache_dict = self.__dict__.setdefault('_memoize_method_dct', {})
         dct = cache_dict.setdefault(method, {})
@@ -90,31 +144,3 @@ def memoize_method(method):
             dct[key] = result
             return result
     return wrapper
-
-
-def _invalidate_star_import_cache_module(module, only_main=False):
-    """ Important if some new modules are being reparsed """
-    try:
-        t, modules = _time_caches['star_import_cache_validity'][module]
-    except KeyError:
-        pass
-    else:
-        del _time_caches['star_import_cache_validity'][module]
-
-        # This stuff was part of load_parser. However since we're most likely
-        # not going to use star import caching anymore, just ignore it.
-        #else:
-            # In case there is already a module cached and this module
-            # has to be reparsed, we also need to invalidate the import
-            # caches.
-        #    _invalidate_star_import_cache_module(parser_cache_item.parser.module)
-
-
-def invalidate_star_import_cache(path):
-    """On success returns True."""
-    try:
-        parser_cache_item = parser_cache[path]
-    except KeyError:
-        pass
-    else:
-        _invalidate_star_import_cache_module(parser_cache_item.parser.module)

jedi/common/__init__.py

@@ -0,0 +1 @@
+from jedi.common.context import BaseContextSet, BaseContext

jedi/common/context.py

@@ -0,0 +1,73 @@
+class BaseContext(object):
+    def __init__(self, evaluator, parent_context=None):
+        self.evaluator = evaluator
+        self.parent_context = parent_context
+
+    def get_root_context(self):
+        context = self
+        while True:
+            if context.parent_context is None:
+                return context
+            context = context.parent_context
+
+
+class BaseContextSet(object):
+    def __init__(self, iterable):
+        self._set = frozenset(iterable)
+        for context in iterable:
+            assert not isinstance(context, BaseContextSet)
+
+    @classmethod
+    def _from_frozen_set(cls, frozenset_):
+        self = cls.__new__(cls)
+        self._set = frozenset_
+        return self
+
+    @classmethod
+    def from_sets(cls, sets):
+        """
+        Used to work with an iterable of set.
+        """
+        aggregated = set()
+        for set_ in sets:
+            if isinstance(set_, BaseContextSet):
+                aggregated |= set_._set
+            else:
+                aggregated |= frozenset(set_)
+        return cls._from_frozen_set(frozenset(aggregated))
+
+    def __or__(self, other):
+        return self._from_frozen_set(self._set | other._set)
+
+    def __and__(self, other):
+        return self._from_frozen_set(self._set & other._set)
+
+    def __iter__(self):
+        for element in self._set:
+            yield element
+
+    def __bool__(self):
+        return bool(self._set)
+
+    def __len__(self):
+        return len(self._set)
+
+    def __repr__(self):
+        return 'S{%s}' % (', '.join(str(s) for s in self._set))
+
+    def filter(self, filter_func):
+        return self.__class__(filter(filter_func, self._set))
+
+    def __getattr__(self, name):
+        def mapper(*args, **kwargs):
+            return self.from_sets(
+                getattr(context, name)(*args, **kwargs)
+                for context in self._set
+            )
+        return mapper
+
+    def __eq__(self, other):
+        return self._set == other._set
+
+    def __hash__(self):
+        return hash(self._set)

jedi/common/utils.py

@@ -0,0 +1,26 @@
+import os
+from contextlib import contextmanager
+
+
+def traverse_parents(path, include_current=False):
+    if not include_current:
+        path = os.path.dirname(path)
+
+    previous = None
+    while previous != path:
+        yield path
+        previous = path
+        path = os.path.dirname(path)
+
+
+@contextmanager
+def monkeypatch(obj, attribute_name, new_value):
+    """
+    Like pytest's monkeypatch, but as a context manager.
+    """
+    old_value = getattr(obj, attribute_name)
+    try:
+        setattr(obj, attribute_name, new_value)
+        yield
+    finally:
+        setattr(obj, attribute_name, old_value)

jedi/debug.py

@@ -1,18 +1,20 @@
-from jedi._compatibility import encoding, is_py3, u
-import inspect
 import os
 import time
+from contextlib import contextmanager
+
+from jedi._compatibility import encoding, is_py3, u
+
+_inited = False
+
 
 def _lazy_colorama_init():
     """
-    Lazily init colorama if necessary, not to screw up stdout is debug not
-    enabled.
+    Lazily init colorama if necessary, not to screw up stdout if debugging is
+    not enabled.
 
     This version of the function does nothing.
     """
-    pass
 
-_inited=False
 
 try:
     if os.name == 'nt':
@@ -22,7 +24,8 @@ try:
         # Use colorama for nicer console output.
         from colorama import Fore, init
         from colorama import initialise
-        def _lazy_colorama_init():
+
+        def _lazy_colorama_init():  # noqa: F811
             """
             Lazily init colorama if necessary, not to screw up stdout is
             debug not enabled.
@@ -36,7 +39,7 @@ try:
                 # need this.
                 initialise.atexit_done = True
                 try:
-                    init()
+                    init(strip=False)
                 except Exception:
                     # Colorama fails with initializing under vim and is buggy in
                     # version 0.3.6.
@@ -50,6 +53,7 @@ except ImportError:
         YELLOW = ''
         MAGENTA = ''
         RESET = ''
+        BLUE = ''
 
 NOTICE = object()
 WARNING = object()
@@ -61,7 +65,6 @@ enable_notice = False
 
 # callback, interface: level, str
 debug_function = None
-ignored_modules = ['jedi.parser']
 _debug_indent = 0
 _start_time = time.time()
 
@@ -75,15 +78,25 @@ def reset_time():
 def increase_indent(func):
     """Decorator for makin """
     def wrapper(*args, **kwargs):
-        global _debug_indent
-        _debug_indent += 1
-        try:
+        with increase_indent_cm():
             return func(*args, **kwargs)
-        finally:
-            _debug_indent -= 1
     return wrapper
 
 
+@contextmanager
+def increase_indent_cm(title=None):
+    global _debug_indent
+    if title:
+        dbg('Start: ' + title, color='MAGENTA')
+    _debug_indent += 1
+    try:
+        yield
+    finally:
+        _debug_indent -= 1
+        if title:
+            dbg('End: ' + title, color='MAGENTA')
+
+
 def dbg(message, *args, **kwargs):
     """ Looks at the stack, to see if a debug message should be printed. """
     # Python 2 compatibility, because it doesn't understand default args
@@ -91,12 +104,9 @@ def dbg(message, *args, **kwargs):
     assert color
 
     if debug_function and enable_notice:
-        frm = inspect.stack()[1]
-        mod = inspect.getmodule(frm[0])
-        if not (mod.__name__ in ignored_modules):
-            i = ' ' * _debug_indent
-            _lazy_colorama_init()
-            debug_function(color, i + 'dbg: ' + message % tuple(u(repr(a)) for a in args))
+        i = ' ' * _debug_indent
+        _lazy_colorama_init()
+        debug_function(color, i + 'dbg: ' + message % tuple(u(repr(a)) for a in args))
 
 
 def warning(message, *args, **kwargs):

jedi/evaluate/__init__.py

@@ -12,29 +12,32 @@ Evaluation of Python code in |jedi| is based on three assumptions:
 * The programmer is not a total dick, e.g. like `this
   <https://github.com/davidhalter/jedi/issues/24>`_ :-)
 
-The actual algorithm is based on a principle called lazy evaluation. If you
-don't know about it, google it.  That said, the typical entry point for static
-analysis is calling ``eval_statement``. There's separate logic for
-autocompletion in the API, the evaluator is all about evaluating an expression.
+The actual algorithm is based on a principle called lazy evaluation.  That
+said, the typical entry point for static analysis is calling
+``eval_expr_stmt``. There's separate logic for autocompletion in the API, the
+evaluator is all about evaluating an expression.
 
-Now you need to understand what follows after ``eval_statement``. Let's
+TODO this paragraph is not what jedi does anymore, it's similar, but not the
+same.
+
+Now you need to understand what follows after ``eval_expr_stmt``. Let's
 make an example::
 
     import datetime
     datetime.date.toda# <-- cursor here
 
 First of all, this module doesn't care about completion. It really just cares
-about ``datetime.date``. At the end of the procedure ``eval_statement`` will
+about ``datetime.date``. At the end of the procedure ``eval_expr_stmt`` will
 return the ``date`` class.
 
 To *visualize* this (simplified):
 
-- ``Evaluator.eval_statement`` doesn't do much, because there's no assignment.
-- ``Evaluator.eval_element`` cares for resolving the dotted path
+- ``Evaluator.eval_expr_stmt`` doesn't do much, because there's no assignment.
+- ``Context.eval_node`` cares for resolving the dotted path
 - ``Evaluator.find_types`` searches for global definitions of datetime, which
   it finds in the definition of an import, by scanning the syntax tree.
 - Using the import logic, the datetime module is found.
-- Now ``find_types`` is called again by ``eval_element`` to find ``date``
+- Now ``find_types`` is called again by ``eval_node`` to find ``date``
   inside the datetime module.
 
 Now what would happen if we wanted ``datetime.date.foo.bar``? Two more
@@ -46,7 +49,7 @@ What if the import would contain another ``ExprStmt`` like this::
     from foo import bar
     Date = bar.baz
 
-Well... You get it. Just another ``eval_statement`` recursion. It's really
+Well... You get it. Just another ``eval_expr_stmt`` recursion. It's really
 easy. Python can obviously get way more complicated then this. To understand
 tuple assignments, list comprehensions and everything else, a lot more code had
 to be written.
@@ -59,143 +62,112 @@ I need to mention now that lazy evaluation is really good because it
 only *evaluates* what needs to be *evaluated*. All the statements and modules
 that are not used are just being ignored.
 """
+from parso.python import tree
+import parso
+from parso import python_bytes_to_unicode
+from jedi.file_io import FileIO
 
-import copy
-import sys
-
-from jedi.parser import tree
 from jedi import debug
-from jedi.common import unite
-from jedi.evaluate import representation as er
+from jedi import parser_utils
+from jedi.evaluate.utils import unite
 from jedi.evaluate import imports
 from jedi.evaluate import recursion
-from jedi.evaluate import iterable
-from jedi.evaluate.cache import memoize_default
-from jedi.evaluate import stdlib
-from jedi.evaluate import finder
-from jedi.evaluate import compiled
-from jedi.evaluate import precedence
-from jedi.evaluate import param
+from jedi.evaluate.cache import evaluator_function_cache
 from jedi.evaluate import helpers
-from jedi.evaluate import pep0484
-from jedi.evaluate.filters import TreeNameDefinition, ParamName
-from jedi.evaluate.instance import AnonymousInstance, BoundMethod
+from jedi.evaluate.names import TreeNameDefinition, ParamName
+from jedi.evaluate.base_context import ContextualizedName, ContextualizedNode, \
+    ContextSet, NO_CONTEXTS, iterate_contexts
+from jedi.evaluate.context import ClassContext, FunctionContext, \
+    AnonymousInstance, BoundMethod
+from jedi.evaluate.context.iterable import CompForContext
+from jedi.evaluate.syntax_tree import eval_trailer, eval_expr_stmt, \
+    eval_node, check_tuple_assignments
+from jedi.plugins import plugin_manager
 
 
 class Evaluator(object):
-    def __init__(self, grammar, sys_path=None):
-        self.grammar = grammar
+    def __init__(self, project, environment=None, script_path=None):
+        if environment is None:
+            environment = project.get_environment()
+        self.environment = environment
+        self.script_path = script_path
+        self.compiled_subprocess = environment.get_evaluator_subprocess(self)
+        self.grammar = environment.get_grammar()
+
+        self.latest_grammar = parso.load_grammar(version='3.7')
         self.memoize_cache = {}  # for memoize decorators
-        # To memorize modules -> equals `sys.modules`.
-        self.modules = {}  # like `sys.modules`.
+        self.module_cache = imports.ModuleCache()  # does the job of `sys.modules`.
+        self.stub_module_cache = {}  # Dict[Tuple[str, ...], Optional[ModuleContext]]
         self.compiled_cache = {}  # see `evaluate.compiled.create()`
-        self.mixed_cache = {}  # see `evaluate.compiled.mixed.create()`
+        self.inferred_element_counts = {}
+        self.mixed_cache = {}  # see `evaluate.compiled.mixed._create()`
         self.analysis = []
         self.dynamic_params_depth = 0
         self.is_analysis = False
-        self.python_version = sys.version_info[:2]
-
-        if sys_path is None:
-            sys_path = sys.path
-        self.sys_path = copy.copy(sys_path)
-        try:
-            self.sys_path.remove('')
-        except ValueError:
-            pass
+        self.project = project
+        self.access_cache = {}
+        self.allow_descriptor_getattr = False
 
         self.reset_recursion_limitations()
+        self.allow_different_encoding = True
 
-        # Constants
-        self.BUILTINS = compiled.get_special_object(self, 'BUILTINS')
+    def import_module(self, import_names, parent_module_context=None,
+                      sys_path=None, prefer_stubs=True):
+        if sys_path is None:
+            sys_path = self.get_sys_path()
+        return imports.import_module(self, import_names, parent_module_context,
+                                     sys_path, prefer_stubs=prefer_stubs)
+
+    @staticmethod
+    @plugin_manager.decorate()
+    def execute(context, arguments):
+        debug.dbg('execute: %s %s', context, arguments)
+        with debug.increase_indent_cm():
+            context_set = context.py__call__(arguments=arguments)
+        debug.dbg('execute result: %s in %s', context_set, context)
+        return context_set
+
+    @property
+    @evaluator_function_cache()
+    def builtins_module(self):
+        module_name = u'builtins'
+        if self.environment.version_info.major == 2:
+            module_name = u'__builtin__'
+        builtins_module, = self.import_module((module_name,), sys_path=())
+        return builtins_module
+
+    @property
+    @evaluator_function_cache()
+    def typing_module(self):
+        typing_module, = self.import_module((u'typing',))
+        return typing_module
 
     def reset_recursion_limitations(self):
         self.recursion_detector = recursion.RecursionDetector()
         self.execution_recursion_detector = recursion.ExecutionRecursionDetector(self)
 
-    def find_types(self, context, name_or_str, name_context, position=None,
-                   search_global=False, is_goto=False):
-        """
-        This is the search function. The most important part to debug.
-        `remove_statements` and `filter_statements` really are the core part of
-        this completion.
-
-        :param position: Position of the last statement -> tuple of line, column
-        :return: List of Names. Their parents are the types.
-        """
-        f = finder.NameFinder(self, context, name_context, name_or_str, position)
-        filters = f.get_filters(search_global)
-        if is_goto:
-            return f.filter_name(filters)
-        return f.find(filters, attribute_lookup=not search_global)
-
-    def eval_statement(self, context, stmt, seek_name=None):
-        with recursion.execution_allowed(self, stmt) as allowed:
-            if allowed or context.get_root_context() == self.BUILTINS:
-                return self._eval_stmt(context, stmt, seek_name)
-        return set()
-
-    #@memoize_default(default=[], evaluator_is_first_arg=True)
-    @debug.increase_indent
-    def _eval_stmt(self, context, stmt, seek_name=None):
-        """
-        The starting point of the completion. A statement always owns a call
-        list, which are the calls, that a statement does. In case multiple
-        names are defined in the statement, `seek_name` returns the result for
-        this name.
-
-        :param stmt: A `tree.ExprStmt`.
-        """
-        debug.dbg('eval_statement %s (%s)', stmt, seek_name)
-        rhs = stmt.get_rhs()
-        types = self.eval_element(context, rhs)
-
-        if seek_name:
-            types = finder.check_tuple_assignments(self, types, seek_name)
-
-        first_operation = stmt.first_operation()
-        if first_operation not in ('=', None) and first_operation.type == 'operator':
-            # `=` is always the last character in aug assignments -> -1
-            operator = copy.copy(first_operation)
-            operator.value = operator.value[:-1]
-            name = str(stmt.get_defined_names()[0])
-            left = context.py__getattribute__(
-                name, position=stmt.start_pos, search_global=True)
-
-            for_stmt = stmt.get_parent_until(tree.ForStmt)
-            if isinstance(for_stmt, tree.ForStmt) and types \
-                    and for_stmt.defines_one_name():
-                # Iterate through result and add the values, that's possible
-                # only in for loops without clutter, because they are
-                # predictable. Also only do it, if the variable is not a tuple.
-                node = for_stmt.get_input_node()
-                for_iterables = self.eval_element(context, node)
-                ordered = list(iterable.py__iter__(self, for_iterables, node))
-
-                for lazy_context in ordered:
-                    dct = {str(for_stmt.children[1]): lazy_context.infer()}
-                    with helpers.predefine_names(context, for_stmt, dct):
-                        t = self.eval_element(context, rhs)
-                        left = precedence.calculate(self, context, left, operator, t)
-                types = left
-            else:
-                types = precedence.calculate(self, context, left, operator, types)
-        debug.dbg('eval_statement result %s', types)
-        return types
+    def get_sys_path(self, **kwargs):
+        """Convenience function"""
+        return self.project._get_sys_path(self, environment=self.environment, **kwargs)
 
     def eval_element(self, context, element):
-        if isinstance(context, iterable.CompForContext):
-            return self._eval_element_not_cached(context, element)
+        if isinstance(context, CompForContext):
+            return eval_node(context, element)
 
         if_stmt = element
         while if_stmt is not None:
             if_stmt = if_stmt.parent
             if if_stmt.type in ('if_stmt', 'for_stmt'):
                 break
-            if if_stmt.is_scope():
+            if parser_utils.is_scope(if_stmt):
                 if_stmt = None
                 break
         predefined_if_name_dict = context.predefined_names.get(if_stmt)
-        if predefined_if_name_dict is None and if_stmt and if_stmt.type == 'if_stmt':
+        # TODO there's a lot of issues with this one. We actually should do
+        # this in a different way. Caching should only be active in certain
+        # cases and this all sucks.
+        if predefined_if_name_dict is None and if_stmt \
+                and if_stmt.type == 'if_stmt' and self.is_analysis:
             if_stmt_test = if_stmt.children[1]
             name_dicts = [{}]
             # If we already did a check, we don't want to do it again -> If
@@ -207,8 +179,8 @@ class Evaluator(object):
                 # names in the suite.
                 if_names = helpers.get_names_of_node(if_stmt_test)
                 element_names = helpers.get_names_of_node(element)
-                str_element_names = [str(e) for e in element_names]
-                if any(str(i) in str_element_names for i in if_names):
+                str_element_names = [e.value for e in element_names]
+                if any(i.value in str_element_names for i in if_names):
                     for if_name in if_names:
                         definitions = self.goto_definitions(context, if_name)
                         # Every name that has multiple different definitions
@@ -229,23 +201,23 @@ class Evaluator(object):
                                 new_name_dicts = list(original_name_dicts)
                                 for i, name_dict in enumerate(new_name_dicts):
                                     new_name_dicts[i] = name_dict.copy()
-                                    new_name_dicts[i][str(if_name)] = set([definition])
+                                    new_name_dicts[i][if_name.value] = ContextSet([definition])
 
                                 name_dicts += new_name_dicts
                         else:
                             for name_dict in name_dicts:
-                                name_dict[str(if_name)] = definitions
+                                name_dict[if_name.value] = definitions
             if len(name_dicts) > 1:
-                result = set()
+                result = NO_CONTEXTS
                 for name_dict in name_dicts:
                     with helpers.predefine_names(context, if_stmt, name_dict):
-                        result |= self._eval_element_not_cached(context, element)
+                        result |= eval_node(context, element)
                 return result
             else:
                 return self._eval_element_if_evaluated(context, element)
         else:
             if predefined_if_name_dict:
-                return self._eval_element_not_cached(context, element)
+                return eval_node(context, element)
             else:
                 return self._eval_element_if_evaluated(context, element)
 
@@ -258,266 +230,137 @@ class Evaluator(object):
             parent = parent.parent
             predefined_if_name_dict = context.predefined_names.get(parent)
             if predefined_if_name_dict is not None:
-                return self._eval_element_not_cached(context, element)
+                return eval_node(context, element)
         return self._eval_element_cached(context, element)
 
-    @memoize_default(default=set(), evaluator_is_first_arg=True)
+    @evaluator_function_cache(default=NO_CONTEXTS)
     def _eval_element_cached(self, context, element):
-        return self._eval_element_not_cached(context, element)
-
-    @debug.increase_indent
-    def _eval_element_not_cached(self, context, element):
-        debug.dbg('eval_element %s@%s', element, element.start_pos)
-        types = set()
-        if isinstance(element, (tree.Name, tree.Literal)) or element.type == 'atom':
-            types = self.eval_atom(context, element)
-        elif isinstance(element, tree.Keyword):
-            # For False/True/None
-            if element.value in ('False', 'True', 'None'):
-                types.add(compiled.builtin_from_name(self, element.value))
-            # else: print e.g. could be evaluated like this in Python 2.7
-        elif isinstance(element, tree.Lambda):
-            types = set([er.FunctionContext(self, context, element)])
-        elif element.type == 'expr_stmt':
-            types = self.eval_statement(context, element)
-        elif element.type in ('power', 'atom_expr'):
-            first_child = element.children[0]
-            if not (first_child.type == 'keyword' and first_child.value == 'await'):
-                types = self.eval_atom(context, first_child)
-                for trailer in element.children[1:]:
-                    if trailer == '**':  # has a power operation.
-                        right = self.eval_element(context, element.children[2])
-                        types = set(precedence.calculate(self, context, types, trailer, right))
-                        break
-                    types = self.eval_trailer(context, types, trailer)
-        elif element.type in ('testlist_star_expr', 'testlist',):
-            # The implicit tuple in statements.
-            types = set([iterable.SequenceLiteralContext(self, context, element)])
-        elif element.type in ('not_test', 'factor'):
-            types = self.eval_element(context, element.children[-1])
-            for operator in element.children[:-1]:
-                types = set(precedence.factor_calculate(self, types, operator))
-        elif element.type == 'test':
-            # `x if foo else y` case.
-            types = (self.eval_element(context, element.children[0]) |
-                     self.eval_element(context, element.children[-1]))
-        elif element.type == 'operator':
-            # Must be an ellipsis, other operators are not evaluated.
-            assert element.value == '...'
-            types = set([compiled.create(self, Ellipsis)])
-        elif element.type == 'dotted_name':
-            types = self.eval_atom(context, element.children[0])
-            for next_name in element.children[2::2]:
-                # TODO add search_global=True?
-                types = unite(
-                    typ.py__getattribute__(next_name, name_context=context)
-                    for typ in types
-                )
-            types = types
-        elif element.type == 'eval_input':
-            types = self._eval_element_not_cached(context, element.children[0])
-        elif element.type == 'annassign':
-            print(element.children[1])
-            types = pep0484._evaluate_for_annotation(context, element.children[1])
-            print('xxx')
-        else:
-            types = precedence.calculate_children(self, context, element.children)
-        debug.dbg('eval_element result %s', types)
-        return types
-
-    def eval_atom(self, context, atom):
-        """
-        Basically to process ``atom`` nodes. The parser sometimes doesn't
-        generate the node (because it has just one child). In that case an atom
-        might be a name or a literal as well.
-        """
-        if isinstance(atom, tree.Name):
-            # This is the first global lookup.
-            stmt = atom.get_definition()
-            if isinstance(stmt, tree.CompFor):
-                stmt = stmt.get_parent_until((tree.ClassOrFunc, tree.ExprStmt))
-            if stmt.type != 'expr_stmt':
-                # We only need to adjust the start_pos for statements, because
-                # there the name cannot be used.
-                stmt = atom
-            return context.py__getattribute__(
-                name_or_str=atom,
-                position=stmt.start_pos,
-                search_global=True
-            )
-        elif isinstance(atom, tree.Literal):
-            return set([compiled.create(self, atom.eval())])
-        else:
-            c = atom.children
-            if c[0].type == 'string':
-                # Will be one string.
-                types = self.eval_atom(context, c[0])
-                for string in c[1:]:
-                    right = self.eval_atom(context, string)
-                    types = precedence.calculate(self, context, types, '+', right)
-                return types
-            # Parentheses without commas are not tuples.
-            elif c[0] == '(' and not len(c) == 2 \
-                    and not(c[1].type == 'testlist_comp' and
-                            len(c[1].children) > 1):
-                return self.eval_element(context, c[1])
-
-            try:
-                comp_for = c[1].children[1]
-            except (IndexError, AttributeError):
-                pass
-            else:
-                if comp_for == ':':
-                    # Dict comprehensions have a colon at the 3rd index.
-                    try:
-                        comp_for = c[1].children[3]
-                    except IndexError:
-                        pass
-
-                if comp_for.type == 'comp_for':
-                    return set([iterable.Comprehension.from_atom(self, context, atom)])
-
-            # It's a dict/list/tuple literal.
-            array_node = c[1]
-            try:
-                array_node_c = array_node.children
-            except AttributeError:
-                array_node_c = []
-            if c[0] == '{' and (array_node == '}' or ':' in array_node_c):
-                context = iterable.DictLiteralContext(self, context, atom)
-            else:
-                context = iterable.SequenceLiteralContext(self, context, atom)
-            return set([context])
-
-    def eval_trailer(self, context, types, trailer):
-        trailer_op, node = trailer.children[:2]
-        if node == ')':  # `arglist` is optional.
-            node = ()
-
-        new_types = set()
-        if trailer_op == '[':
-            new_types |= iterable.py__getitem__(self, context, types, trailer)
-        else:
-            for typ in types:
-                debug.dbg('eval_trailer: %s in scope %s', trailer, typ)
-                if trailer_op == '.':
-                    new_types |= typ.py__getattribute__(
-                        name_context=context,
-                        name_or_str=node
-                    )
-                elif trailer_op == '(':
-                    arguments = param.TreeArguments(self, context, node, trailer)
-                    new_types |= self.execute(typ, arguments)
-        return new_types
-
-    @debug.increase_indent
-    def execute(self, obj, arguments):
-        if not isinstance(arguments, param.AbstractArguments):
-            raise NotImplementedError
-            arguments = param.Arguments(self, arguments)
-
-        if self.is_analysis:
-            arguments.eval_all()
-
-        debug.dbg('execute: %s %s', obj, arguments)
-        try:
-            # Some stdlib functions like super(), namedtuple(), etc. have been
-            # hard-coded in Jedi to support them.
-            return stdlib.execute(self, obj, arguments)
-        except stdlib.NotInStdLib:
-            pass
-
-        try:
-            func = obj.py__call__
-        except AttributeError:
-            debug.warning("no execution possible %s", obj)
-            return set()
-        else:
-            types = func(arguments)
-            debug.dbg('execute result: %s in %s', types, obj)
-            return types
+        return eval_node(context, element)
 
     def goto_definitions(self, context, name):
-        def_ = name.get_definition()
-        is_simple_name = name.parent.type not in ('power', 'trailer')
-        if is_simple_name:
-            if name.parent.type == 'classdef' and name.parent.name == name:
-                return [er.ClassContext(self, name.parent, context)]
-            elif name.parent.type == 'funcdef':
-                return [er.FunctionContext(self, context, name.parent)]
-            elif name.parent.type == 'file_input':
-                raise NotImplementedError
-            if def_.type == 'expr_stmt' and name in def_.get_defined_names():
-                return self.eval_statement(context, def_, name)
-            elif def_.type == 'for_stmt':
-                container_types = self.eval_element(context, def_.children[3])
-                for_types = iterable.py__iter__types(self, container_types, def_.children[3])
-                return finder.check_tuple_assignments(self, for_types, name)
-            elif def_.type in ('import_from', 'import_name'):
+        def_ = name.get_definition(import_name_always=True)
+        if def_ is not None:
+            type_ = def_.type
+            is_classdef = type_ == 'classdef'
+            if is_classdef or type_ == 'funcdef':
+                if is_classdef:
+                    c = ClassContext(self, context, name.parent)
+                else:
+                    c = FunctionContext.from_context(context, name.parent)
+                return ContextSet([c])
+
+            if type_ == 'expr_stmt':
+                is_simple_name = name.parent.type not in ('power', 'trailer')
+                if is_simple_name:
+                    return eval_expr_stmt(context, def_, name)
+            if type_ == 'for_stmt':
+                container_types = context.eval_node(def_.children[3])
+                cn = ContextualizedNode(context, def_.children[3])
+                for_types = iterate_contexts(container_types, cn)
+                c_node = ContextualizedName(context, name)
+                return check_tuple_assignments(self, c_node, for_types)
+            if type_ in ('import_from', 'import_name'):
                 return imports.infer_import(context, name)
+        else:
+            result = self._follow_error_node_imports_if_possible(context, name)
+            if result is not None:
+                return result
 
         return helpers.evaluate_call_of_leaf(context, name)
 
+    def _follow_error_node_imports_if_possible(self, context, name):
+        error_node = tree.search_ancestor(name, 'error_node')
+        if error_node is not None:
+            # Get the first command start of a started simple_stmt. The error
+            # node is sometimes a small_stmt and sometimes a simple_stmt. Check
+            # for ; leaves that start a new statements.
+            start_index = 0
+            for index, n in enumerate(error_node.children):
+                if n.start_pos > name.start_pos:
+                    break
+                if n == ';':
+                    start_index = index + 1
+            nodes = error_node.children[start_index:]
+            first_name = nodes[0].get_first_leaf().value
+
+            # Make it possible to infer stuff like `import foo.` or
+            # `from foo.bar`.
+            if first_name in ('from', 'import'):
+                is_import_from = first_name == 'from'
+                level, names = helpers.parse_dotted_names(
+                    nodes,
+                    is_import_from=is_import_from,
+                    until_node=name,
+                )
+                return imports.Importer(self, names, context.get_root_context(), level).follow()
+        return None
+
     def goto(self, context, name):
-        stmt = name.get_definition()
+        definition = name.get_definition(import_name_always=True)
+        if definition is not None:
+            type_ = definition.type
+            if type_ == 'expr_stmt':
+                # Only take the parent, because if it's more complicated than just
+                # a name it's something you can "goto" again.
+                is_simple_name = name.parent.type not in ('power', 'trailer')
+                if is_simple_name:
+                    return [TreeNameDefinition(context, name)]
+            elif type_ == 'param':
+                return [ParamName(context, name)]
+            elif type_ in ('import_from', 'import_name'):
+                module_names = imports.infer_import(context, name, is_goto=True)
+                return module_names
+            else:
+                return [TreeNameDefinition(context, name)]
+        else:
+            contexts = self._follow_error_node_imports_if_possible(context, name)
+            if contexts is not None:
+                return [context.name for context in contexts]
+
         par = name.parent
-        if par.type == 'argument' and par.children[1] == '=' and par.children[0] == name:
+        node_type = par.type
+        if node_type == 'argument' and par.children[1] == '=' and par.children[0] == name:
             # Named param goto.
             trailer = par.parent
             if trailer.type == 'arglist':
                 trailer = trailer.parent
             if trailer.type != 'classdef':
                 if trailer.type == 'decorator':
-                    types = self.eval_element(context, trailer.children[1])
+                    context_set = context.eval_node(trailer.children[1])
                 else:
                     i = trailer.parent.children.index(trailer)
                     to_evaluate = trailer.parent.children[:i]
-                    types = self.eval_element(context, to_evaluate[0])
+                    if to_evaluate[0] == 'await':
+                        to_evaluate.pop(0)
+                    context_set = context.eval_node(to_evaluate[0])
                     for trailer in to_evaluate[1:]:
-                        types = self.eval_trailer(context, types, trailer)
+                        context_set = eval_trailer(context, context_set, trailer)
                 param_names = []
-                for typ in types:
-                    try:
-                        get_param_names = typ.get_param_names
-                    except AttributeError:
-                        pass
-                    else:
-                        for param_name in get_param_names():
+                for context in context_set:
+                    for signature in context.get_signatures():
+                        for param_name in signature.get_param_names():
                             if param_name.string_name == name.value:
                                 param_names.append(param_name)
                 return param_names
-        elif isinstance(par, tree.ExprStmt) and name in par.get_defined_names():
-            # Only take the parent, because if it's more complicated than just
-            # a name it's something you can "goto" again.
-            return [TreeNameDefinition(context, name)]
-        elif par.type == 'param' and par.name:
-            return [ParamName(context, name)]
-        elif isinstance(par, (tree.Param, tree.Function, tree.Class)) and par.name is name:
-            return [TreeNameDefinition(context, name)]
-        elif isinstance(stmt, tree.Import):
-            module_names = imports.infer_import(context, name, is_goto=True)
-            return module_names
-        elif par.type == 'dotted_name':  # Is a decorator.
+        elif node_type == 'dotted_name':  # Is a decorator.
             index = par.children.index(name)
             if index > 0:
                 new_dotted = helpers.deep_ast_copy(par)
                 new_dotted.children[index - 1:] = []
-                values = self.eval_element(context, new_dotted)
+                values = context.eval_node(new_dotted)
                 return unite(
                     value.py__getattribute__(name, name_context=context, is_goto=True)
                     for value in values
                 )
 
-        if par.type == 'trailer' and par.children[0] == '.':
+        if node_type == 'trailer' and par.children[0] == '.':
             values = helpers.evaluate_call_of_leaf(context, name, cut_own_trailer=True)
-            return unite(
-                value.py__getattribute__(name, name_context=context, is_goto=True)
-                for value in values
-            )
+            return values.py__getattribute__(name, name_context=context, is_goto=True)
         else:
-            if stmt.type != 'expr_stmt':
-                # We only need to adjust the start_pos for statements, because
-                # there the name cannot be used.
+            stmt = tree.search_ancestor(
+                name, 'expr_stmt', 'lambdef'
+            ) or name
+            if stmt.type == 'lambdef':
                 stmt = name
             return context.py__getattribute__(
                 name,
@@ -530,61 +373,71 @@ class Evaluator(object):
             while True:
                 node = node.parent
 
-                if node.is_scope():
+                if parser_utils.is_scope(node):
                     return node
                 elif node.type in ('argument', 'testlist_comp'):
-                    if node.children[1].type == 'comp_for':
+                    if node.children[1].type in ('comp_for', 'sync_comp_for'):
                         return node.children[1]
                 elif node.type == 'dictorsetmaker':
                     for n in node.children[1:4]:
                         # In dictionaries it can be pretty much anything.
-                        if n.type == 'comp_for':
+                        if n.type in ('comp_for', 'sync_comp_for'):
                             return n
 
-        def from_scope_node(scope_node, child_is_funcdef=None, is_nested=True, node_is_object=False):
+        def from_scope_node(scope_node, is_nested=True, node_is_object=False):
             if scope_node == base_node:
                 return base_context
 
-            is_funcdef = scope_node.type in ('funcdef', 'lambda')
-            parent_scope = scope_node.get_parent_scope()
-            parent_context = from_scope_node(parent_scope, child_is_funcdef=is_funcdef)
+            is_funcdef = scope_node.type in ('funcdef', 'lambdef')
+            parent_scope = parser_utils.get_parent_scope(scope_node)
+            parent_context = from_scope_node(parent_scope)
 
             if is_funcdef:
-                if isinstance(parent_context, AnonymousInstance):
+                func = FunctionContext.from_context(parent_context, scope_node)
+                if parent_context.is_class():
+                    instance = AnonymousInstance(
+                        self, parent_context.parent_context, parent_context)
                     func = BoundMethod(
-                        self, parent_context, parent_context.class_context,
-                        parent_context.parent_context, scope_node
-                    )
-                else:
-                    func = er.FunctionContext(
-                        self,
-                        parent_context,
-                        scope_node
+                        instance=instance,
+                        function=func
                     )
+
                 if is_nested and not node_is_object:
                     return func.get_function_execution()
                 return func
             elif scope_node.type == 'classdef':
-                class_context = er.ClassContext(self, scope_node, parent_context)
-                if child_is_funcdef:
-                    # anonymous instance
-                    return AnonymousInstance(self, parent_context, class_context)
-                else:
-                    return class_context
-            elif scope_node.type == 'comp_for':
+                return ClassContext(self, parent_context, scope_node)
+            elif scope_node.type in ('comp_for', 'sync_comp_for'):
                 if node.start_pos >= scope_node.children[-1].start_pos:
                     return parent_context
-                return iterable.CompForContext.from_comp_for(parent_context, scope_node)
+                return CompForContext.from_comp_for(parent_context, scope_node)
             raise Exception("There's a scope that was not managed.")
 
         base_node = base_context.tree_node
 
-        if node_is_context and node.is_scope():
+        if node_is_context and parser_utils.is_scope(node):
             scope_node = node
         else:
-            if node.parent.type in ('funcdef', 'classdef'):
-                # When we're on class/function names/leafs that define the
-                # object itself and not its contents.
-                node = node.parent
             scope_node = parent_scope(node)
+            if scope_node.type in ('funcdef', 'classdef'):
+                colon = scope_node.children[scope_node.children.index(':')]
+                if node.start_pos < colon.start_pos:
+                    parent = node.parent
+                    if not (parent.type == 'param' and parent.name == node):
+                        scope_node = parent_scope(scope_node)
         return from_scope_node(scope_node, is_nested=True, node_is_object=node_is_object)
+
+    def parse_and_get_code(self, code=None, path=None, encoding='utf-8',
+                           use_latest_grammar=False, file_io=None, **kwargs):
+        if self.allow_different_encoding:
+            if code is None:
+                if file_io is None:
+                    file_io = FileIO(path)
+                code = file_io.read()
+            code = python_bytes_to_unicode(code, encoding=encoding, errors='replace')
+
+        grammar = self.latest_grammar if use_latest_grammar else self.grammar
+        return grammar.parse(code=code, path=path, file_io=file_io, **kwargs), code
+
+    def parse(self, *args, **kwargs):
+        return self.parse_and_get_code(*args, **kwargs)[0]

jedi/evaluate/analysis.py

@@ -1,11 +1,11 @@
 """
 Module for statical analysis.
 """
-from jedi import debug
-from jedi.parser import tree
-from jedi.evaluate.compiled import CompiledObject
+from parso.python import tree
 
-from jedi.common import unite
+from jedi._compatibility import force_unicode
+from jedi import debug
+from jedi.evaluate.helpers import is_string
 
 
 CODES = {
@@ -82,41 +82,47 @@ def add(node_context, error_name, node, message=None, typ=Error, payload=None):
     if _check_for_exception_catch(node_context, node, exception, payload):
         return
 
-    module_path = node.get_root_node().path
-    instance = typ(error_name, module_path, node.start_pos, message)
-    debug.warning(str(instance), format=False)
-    node_context.evaluator.analysis.append(instance)
+    # TODO this path is probably not right
+    module_context = node_context.get_root_context()
+    module_path = module_context.py__file__()
+    issue_instance = typ(error_name, module_path, node.start_pos, message)
+    debug.warning(str(issue_instance), format=False)
+    node_context.evaluator.analysis.append(issue_instance)
+    return issue_instance
 
 
 def _check_for_setattr(instance):
     """
     Check if there's any setattr method inside an instance. If so, return True.
     """
-    from jedi.evaluate.representation import ModuleContext
     module = instance.get_root_context()
-    if not isinstance(module, ModuleContext):
+    node = module.tree_node
+    if node is None:
+        # If it's a compiled module or doesn't have a tree_node
         return False
 
-    node = module.tree_node
     try:
-        stmts = node.used_names['setattr']
+        stmt_names = node.get_used_names()['setattr']
     except KeyError:
         return False
 
-    return any(node.start_pos < stmt.start_pos < node.end_pos
-               for stmt in stmts)
+    return any(node.start_pos < n.start_pos < node.end_pos
+               # Check if it's a function called setattr.
+               and not (n.parent.type == 'funcdef' and n.parent.name == n)
+               for n in stmt_names)
 
 
 def add_attribute_error(name_context, lookup_context, name):
     message = ('AttributeError: %s has no attribute %s.' % (lookup_context, name))
-    from jedi.evaluate.instance import AbstractInstanceContext, CompiledInstanceName
+    from jedi.evaluate.context.instance import CompiledInstanceName
     # Check for __getattr__/__getattribute__ existance and issue a warning
     # instead of an error, if that happens.
     typ = Error
-    if isinstance(lookup_context, AbstractInstanceContext):
-        slot_names = lookup_context.get_function_slot_names('__getattr__') + \
-            lookup_context.get_function_slot_names('__getattribute__')
+    if lookup_context.is_instance() and not lookup_context.is_compiled():
+        slot_names = lookup_context.get_function_slot_names(u'__getattr__') + \
+            lookup_context.get_function_slot_names(u'__getattribute__')
         for n in slot_names:
+            # TODO do we even get here?
             if isinstance(name, CompiledInstanceName) and \
                     n.parent_context.obj == object:
                 typ = Warning
@@ -138,11 +144,15 @@ def _check_for_exception_catch(node_context, jedi_name, exception, payload=None)
     Returns True if the exception was catched.
     """
     def check_match(cls, exception):
-        try:
-            return isinstance(cls, CompiledObject) and issubclass(exception, cls.obj)
-        except TypeError:
+        if not cls.is_class():
             return False
 
+        for python_cls in exception.mro():
+            if cls.py__name__() == python_cls.__name__ \
+                    and cls.parent_context == cls.evaluator.builtins_module:
+                return True
+        return False
+
     def check_try_for_except(obj, exception):
         # Only nodes in try
         iterator = iter(obj.children)
@@ -153,14 +163,14 @@ def _check_for_exception_catch(node_context, jedi_name, exception, payload=None)
                     and not (branch_type.start_pos < jedi_name.start_pos <= suite.end_pos):
                 return False
 
-        for node in obj.except_clauses():
+        for node in obj.get_except_clause_tests():
             if node is None:
                 return True  # An exception block that catches everything.
             else:
                 except_classes = node_context.eval_node(node)
                 for cls in except_classes:
-                    from jedi.evaluate import iterable
-                    if isinstance(cls, iterable.AbstractSequence) and \
+                    from jedi.evaluate.context import iterable
+                    if isinstance(cls, iterable.Sequence) and \
                             cls.array_type == 'tuple':
                         # multiple exceptions
                         for lazy_context in cls.py__iter__():
@@ -181,7 +191,7 @@ def _check_for_exception_catch(node_context, jedi_name, exception, payload=None)
             assert trailer.type == 'trailer'
             arglist = trailer.children[1]
             assert arglist.type == 'arglist'
-            from jedi.evaluate.param import TreeArguments
+            from jedi.evaluate.arguments import TreeArguments
             args = list(TreeArguments(node_context.evaluator, node_context, arglist).unpack())
             # Arguments should be very simple
             assert len(args) == 2
@@ -189,8 +199,8 @@ def _check_for_exception_catch(node_context, jedi_name, exception, payload=None)
             # Check name
             key, lazy_context = args[1]
             names = list(lazy_context.infer())
-            assert len(names) == 1 and isinstance(names[0], CompiledObject)
-            assert names[0].obj == str(payload[1])
+            assert len(names) == 1 and is_string(names[0])
+            assert force_unicode(names[0].get_safe_value()) == payload[1].value
 
             # Check objects
             key, lazy_context = args[0]
@@ -203,10 +213,10 @@ def _check_for_exception_catch(node_context, jedi_name, exception, payload=None)
     while obj is not None and not isinstance(obj, (tree.Function, tree.Class)):
         if isinstance(obj, tree.Flow):
             # try/except catch check
-            if obj.isinstance(tree.TryStmt) and check_try_for_except(obj, exception):
+            if obj.type == 'try_stmt' and check_try_for_except(obj, exception):
                 return True
             # hasattr check
-            if exception == AttributeError and obj.isinstance(tree.IfStmt, tree.WhileStmt):
+            if exception == AttributeError and obj.type in ('if_stmt', 'while_stmt'):
                 if check_hasattr(obj.children[1], obj.children[3]):
                     return True
         obj = obj.parent

jedi/evaluate/arguments.py

@@ -0,0 +1,382 @@
+import re
+
+from parso.python import tree
+
+from jedi._compatibility import zip_longest
+from jedi import debug
+from jedi.evaluate.utils import PushBackIterator
+from jedi.evaluate import analysis
+from jedi.evaluate.lazy_context import LazyKnownContext, LazyKnownContexts, \
+    LazyTreeContext, get_merged_lazy_context
+from jedi.evaluate.names import ParamName, TreeNameDefinition
+from jedi.evaluate.base_context import NO_CONTEXTS, ContextSet, ContextualizedNode
+from jedi.evaluate.context import iterable
+from jedi.evaluate.cache import evaluator_as_method_param_cache
+from jedi.evaluate.param import get_executed_params_and_issues, ExecutedParam
+
+
+def try_iter_content(types, depth=0):
+    """Helper method for static analysis."""
+    if depth > 10:
+        # It's possible that a loop has references on itself (especially with
+        # CompiledObject). Therefore don't loop infinitely.
+        return
+
+    for typ in types:
+        try:
+            f = typ.py__iter__
+        except AttributeError:
+            pass
+        else:
+            for lazy_context in f():
+                try_iter_content(lazy_context.infer(), depth + 1)
+
+
+class ParamIssue(Exception):
+    pass
+
+
+def repack_with_argument_clinic(string, keep_arguments_param=False, keep_callback_param=False):
+    """
+    Transforms a function or method with arguments to the signature that is
+    given as an argument clinic notation.
+
+    Argument clinic is part of CPython and used for all the functions that are
+    implemented in C (Python 3.7):
+
+        str.split.__text_signature__
+        # Results in: '($self, /, sep=None, maxsplit=-1)'
+    """
+    clinic_args = list(_parse_argument_clinic(string))
+
+    def decorator(func):
+        def wrapper(context, *args, **kwargs):
+            if keep_arguments_param:
+                arguments = kwargs['arguments']
+            else:
+                arguments = kwargs.pop('arguments')
+            if not keep_arguments_param:
+                kwargs.pop('callback', None)
+            try:
+                args += tuple(_iterate_argument_clinic(
+                    context.evaluator,
+                    arguments,
+                    clinic_args
+                ))
+            except ParamIssue:
+                return NO_CONTEXTS
+            else:
+                return func(context, *args, **kwargs)
+
+        return wrapper
+    return decorator
+
+
+def _iterate_argument_clinic(evaluator, arguments, parameters):
+    """Uses a list with argument clinic information (see PEP 436)."""
+    iterator = PushBackIterator(arguments.unpack())
+    for i, (name, optional, allow_kwargs, stars) in enumerate(parameters):
+        if stars == 1:
+            lazy_contexts = []
+            for key, argument in iterator:
+                if key is not None:
+                    iterator.push_back((key, argument))
+                    break
+
+                lazy_contexts.append(argument)
+            yield ContextSet([iterable.FakeSequence(evaluator, u'tuple', lazy_contexts)])
+            lazy_contexts
+            continue
+        elif stars == 2:
+            raise NotImplementedError()
+        key, argument = next(iterator, (None, None))
+        if key is not None:
+            debug.warning('Keyword arguments in argument clinic are currently not supported.')
+            raise ParamIssue
+        if argument is None and not optional:
+            debug.warning('TypeError: %s expected at least %s arguments, got %s',
+                          name, len(parameters), i)
+            raise ParamIssue
+
+        context_set = NO_CONTEXTS if argument is None else argument.infer()
+
+        if not context_set and not optional:
+            # For the stdlib we always want values. If we don't get them,
+            # that's ok, maybe something is too hard to resolve, however,
+            # we will not proceed with the evaluation of that function.
+            debug.warning('argument_clinic "%s" not resolvable.', name)
+            raise ParamIssue
+        yield context_set
+
+
+def _parse_argument_clinic(string):
+    allow_kwargs = False
+    optional = False
+    while string:
+        # Optional arguments have to begin with a bracket. And should always be
+        # at the end of the arguments. This is therefore not a proper argument
+        # clinic implementation. `range()` for exmple allows an optional start
+        # value at the beginning.
+        match = re.match(r'(?:(?:(\[),? ?|, ?|)(\**\w+)|, ?/)\]*', string)
+        string = string[len(match.group(0)):]
+        if not match.group(2):  # A slash -> allow named arguments
+            allow_kwargs = True
+            continue
+        optional = optional or bool(match.group(1))
+        word = match.group(2)
+        stars = word.count('*')
+        word = word[stars:]
+        yield (word, optional, allow_kwargs, stars)
+        if stars:
+            allow_kwargs = True
+
+
+class _AbstractArgumentsMixin(object):
+    def eval_all(self, funcdef=None):
+        """
+        Evaluates all arguments as a support for static analysis
+        (normally Jedi).
+        """
+        for key, lazy_context in self.unpack():
+            types = lazy_context.infer()
+            try_iter_content(types)
+
+    def unpack(self, funcdef=None):
+        raise NotImplementedError
+
+    def get_executed_params_and_issues(self, execution_context):
+        return get_executed_params_and_issues(execution_context, self)
+
+    def get_calling_nodes(self):
+        return []
+
+
+class AbstractArguments(_AbstractArgumentsMixin):
+    context = None
+    argument_node = None
+    trailer = None
+
+
+class AnonymousArguments(AbstractArguments):
+    def get_executed_params_and_issues(self, execution_context):
+        from jedi.evaluate.dynamic import search_params
+        return search_params(
+            execution_context.evaluator,
+            execution_context,
+            execution_context.tree_node
+        ), []
+
+    def __repr__(self):
+        return '%s()' % self.__class__.__name__
+
+
+def unpack_arglist(arglist):
+    if arglist is None:
+        return
+
+    # Allow testlist here as well for Python2's class inheritance
+    # definitions.
+    if not (arglist.type in ('arglist', 'testlist') or (
+            # in python 3.5 **arg is an argument, not arglist
+            (arglist.type == 'argument') and
+            arglist.children[0] in ('*', '**'))):
+        yield 0, arglist
+        return
+
+    iterator = iter(arglist.children)
+    for child in iterator:
+        if child == ',':
+            continue
+        elif child in ('*', '**'):
+            yield len(child.value), next(iterator)
+        elif child.type == 'argument' and \
+                child.children[0] in ('*', '**'):
+            assert len(child.children) == 2
+            yield len(child.children[0].value), child.children[1]
+        else:
+            yield 0, child
+
+
+class TreeArguments(AbstractArguments):
+    def __init__(self, evaluator, context, argument_node, trailer=None):
+        """
+        The argument_node is either a parser node or a list of evaluated
+        objects. Those evaluated objects may be lists of evaluated objects
+        themselves (one list for the first argument, one for the second, etc).
+
+        :param argument_node: May be an argument_node or a list of nodes.
+        """
+        self.argument_node = argument_node
+        self.context = context
+        self._evaluator = evaluator
+        self.trailer = trailer  # Can be None, e.g. in a class definition.
+
+    @classmethod
+    @evaluator_as_method_param_cache()
+    def create_cached(cls, *args, **kwargs):
+        return cls(*args, **kwargs)
+
+    def unpack(self, funcdef=None):
+        named_args = []
+        for star_count, el in unpack_arglist(self.argument_node):
+            if star_count == 1:
+                arrays = self.context.eval_node(el)
+                iterators = [_iterate_star_args(self.context, a, el, funcdef)
+                             for a in arrays]
+                for values in list(zip_longest(*iterators)):
+                    # TODO zip_longest yields None, that means this would raise
+                    # an exception?
+                    yield None, get_merged_lazy_context(
+                        [v for v in values if v is not None]
+                    )
+            elif star_count == 2:
+                arrays = self.context.eval_node(el)
+                for dct in arrays:
+                    for key, values in _star_star_dict(self.context, dct, el, funcdef):
+                        yield key, values
+            else:
+                if el.type == 'argument':
+                    c = el.children
+                    if len(c) == 3:  # Keyword argument.
+                        named_args.append((c[0].value, LazyTreeContext(self.context, c[2]),))
+                    else:  # Generator comprehension.
+                        # Include the brackets with the parent.
+                        sync_comp_for = el.children[1]
+                        if sync_comp_for.type == 'comp_for':
+                            sync_comp_for = sync_comp_for.children[1]
+                        comp = iterable.GeneratorComprehension(
+                            self._evaluator,
+                            defining_context=self.context,
+                            sync_comp_for_node=sync_comp_for,
+                            entry_node=el.children[0],
+                        )
+                        yield None, LazyKnownContext(comp)
+                else:
+                    yield None, LazyTreeContext(self.context, el)
+
+        # Reordering arguments is necessary, because star args sometimes appear
+        # after named argument, but in the actual order it's prepended.
+        for named_arg in named_args:
+            yield named_arg
+
+    def _as_tree_tuple_objects(self):
+        for star_count, argument in unpack_arglist(self.argument_node):
+            default = None
+            if argument.type == 'argument':
+                if len(argument.children) == 3:  # Keyword argument.
+                    argument, default = argument.children[::2]
+            yield argument, default, star_count
+
+    def iter_calling_names_with_star(self):
+        for name, default, star_count in self._as_tree_tuple_objects():
+            # TODO this function is a bit strange. probably refactor?
+            if not star_count or not isinstance(name, tree.Name):
+                continue
+
+            yield TreeNameDefinition(self.context, name)
+
+    def __repr__(self):
+        return '<%s: %s>' % (self.__class__.__name__, self.argument_node)
+
+    def get_calling_nodes(self):
+        from jedi.evaluate.dynamic import DynamicExecutedParams
+        old_arguments_list = []
+        arguments = self
+
+        while arguments not in old_arguments_list:
+            if not isinstance(arguments, TreeArguments):
+                break
+
+            old_arguments_list.append(arguments)
+            for calling_name in reversed(list(arguments.iter_calling_names_with_star())):
+                names = calling_name.goto()
+                if len(names) != 1:
+                    break
+                if not isinstance(names[0], ParamName):
+                    break
+                param = names[0].get_param()
+                if isinstance(param, DynamicExecutedParams):
+                    # For dynamic searches we don't even want to see errors.
+                    return []
+                if not isinstance(param, ExecutedParam):
+                    break
+                if param.var_args is None:
+                    break
+                arguments = param.var_args
+                break
+
+        if arguments.argument_node is not None:
+            return [ContextualizedNode(arguments.context, arguments.argument_node)]
+        if arguments.trailer is not None:
+            return [ContextualizedNode(arguments.context, arguments.trailer)]
+        return []
+
+
+class ValuesArguments(AbstractArguments):
+    def __init__(self, values_list):
+        self._values_list = values_list
+
+    def unpack(self, funcdef=None):
+        for values in self._values_list:
+            yield None, LazyKnownContexts(values)
+
+    def __repr__(self):
+        return '<%s: %s>' % (self.__class__.__name__, self._values_list)
+
+
+class TreeArgumentsWrapper(_AbstractArgumentsMixin):
+    def __init__(self, arguments):
+        self._wrapped_arguments = arguments
+
+    @property
+    def context(self):
+        return self._wrapped_arguments.context
+
+    @property
+    def argument_node(self):
+        return self._wrapped_arguments.argument_node
+
+    @property
+    def trailer(self):
+        return self._wrapped_arguments.trailer
+
+    def unpack(self, func=None):
+        raise NotImplementedError
+
+    def get_calling_nodes(self):
+        return self._wrapped_arguments.get_calling_nodes()
+
+    def __repr__(self):
+        return '<%s: %s>' % (self.__class__.__name__, self._wrapped_arguments)
+
+
+def _iterate_star_args(context, array, input_node, funcdef=None):
+    if not array.py__getattribute__('__iter__'):
+        if funcdef is not None:
+            # TODO this funcdef should not be needed.
+            m = "TypeError: %s() argument after * must be a sequence, not %s" \
+                % (funcdef.name.value, array)
+            analysis.add(context, 'type-error-star', input_node, message=m)
+    try:
+        iter_ = array.py__iter__
+    except AttributeError:
+        pass
+    else:
+        for lazy_context in iter_():
+            yield lazy_context
+
+
+def _star_star_dict(context, array, input_node, funcdef):
+    from jedi.evaluate.context.instance import CompiledInstance
+    if isinstance(array, CompiledInstance) and array.name.string_name == 'dict':
+        # For now ignore this case. In the future add proper iterators and just
+        # make one call without crazy isinstance checks.
+        return {}
+    elif isinstance(array, iterable.Sequence) and array.array_type == 'dict':
+        return array.exact_key_items()
+    else:
+        if funcdef is not None:
+            m = "TypeError: %s argument after ** must be a mapping, not %s" \
+                % (funcdef.name.value, array)
+            analysis.add(context, 'type-error-star-star', input_node, message=m)
+        return {}

jedi/evaluate/base_context.py

@@ -0,0 +1,436 @@
+"""
+Contexts are the "values" that Python would return. However Contexts are at the
+same time also the "contexts" that a user is currently sitting in.
+
+A ContextSet is typically used to specify the return of a function or any other
+static analysis operation. In jedi there are always multiple returns and not
+just one.
+"""
+from functools import reduce
+from operator import add
+from parso.python.tree import ExprStmt, SyncCompFor
+
+from jedi import debug
+from jedi._compatibility import zip_longest, unicode
+from jedi.parser_utils import clean_scope_docstring
+from jedi.common import BaseContextSet, BaseContext
+from jedi.evaluate.helpers import SimpleGetItemNotFound
+from jedi.evaluate.utils import safe_property
+from jedi.evaluate.cache import evaluator_as_method_param_cache
+from jedi.cache import memoize_method
+
+_sentinel = object()
+
+
+class HelperContextMixin(object):
+    def get_root_context(self):
+        context = self
+        while True:
+            if context.parent_context is None:
+                return context
+            context = context.parent_context
+
+    @classmethod
+    @evaluator_as_method_param_cache()
+    def create_cached(cls, *args, **kwargs):
+        return cls(*args, **kwargs)
+
+    def execute(self, arguments):
+        return self.evaluator.execute(self, arguments=arguments)
+
+    def execute_evaluated(self, *value_list):
+        from jedi.evaluate.arguments import ValuesArguments
+        arguments = ValuesArguments([ContextSet([value]) for value in value_list])
+        return self.evaluator.execute(self, arguments)
+
+    def execute_annotation(self):
+        return self.execute_evaluated()
+
+    def gather_annotation_classes(self):
+        return ContextSet([self])
+
+    def merge_types_of_iterate(self, contextualized_node=None, is_async=False):
+        return ContextSet.from_sets(
+            lazy_context.infer()
+            for lazy_context in self.iterate(contextualized_node, is_async)
+        )
+
+    def py__getattribute__(self, name_or_str, name_context=None, position=None,
+                           search_global=False, is_goto=False,
+                           analysis_errors=True):
+        """
+        :param position: Position of the last statement -> tuple of line, column
+        """
+        if name_context is None:
+            name_context = self
+        from jedi.evaluate import finder
+        f = finder.NameFinder(self.evaluator, self, name_context, name_or_str,
+                              position, analysis_errors=analysis_errors)
+        filters = f.get_filters(search_global)
+        if is_goto:
+            return f.filter_name(filters)
+        return f.find(filters, attribute_lookup=not search_global)
+
+    def py__await__(self):
+        await_context_set = self.py__getattribute__(u"__await__")
+        if not await_context_set:
+            debug.warning('Tried to run __await__ on context %s', self)
+        return await_context_set.execute_evaluated()
+
+    def eval_node(self, node):
+        return self.evaluator.eval_element(self, node)
+
+    def create_context(self, node, node_is_context=False, node_is_object=False):
+        return self.evaluator.create_context(self, node, node_is_context, node_is_object)
+
+    def iterate(self, contextualized_node=None, is_async=False):
+        debug.dbg('iterate %s', self)
+        if is_async:
+            from jedi.evaluate.lazy_context import LazyKnownContexts
+            # TODO if no __aiter__ contexts are there, error should be:
+            # TypeError: 'async for' requires an object with __aiter__ method, got int
+            return iter([
+                LazyKnownContexts(
+                    self.py__getattribute__('__aiter__').execute_evaluated()
+                        .py__getattribute__('__anext__').execute_evaluated()
+                        .py__getattribute__('__await__').execute_evaluated()
+                        .py__stop_iteration_returns()
+                )  # noqa
+            ])
+        return self.py__iter__(contextualized_node)
+
+    def is_sub_class_of(self, class_context):
+        for cls in self.py__mro__():
+            if cls.is_same_class(class_context):
+                return True
+        return False
+
+    def is_same_class(self, class2):
+        # Class matching should prefer comparisons that are not this function.
+        if type(class2).is_same_class != HelperContextMixin.is_same_class:
+            return class2.is_same_class(self)
+        return self == class2
+
+
+class Context(HelperContextMixin, BaseContext):
+    """
+    Should be defined, otherwise the API returns empty types.
+    """
+    predefined_names = {}
+    """
+    To be defined by subclasses.
+    """
+    tree_node = None
+
+    @property
+    def api_type(self):
+        # By default just lower name of the class. Can and should be
+        # overwritten.
+        return self.__class__.__name__.lower()
+
+    def py__getitem__(self, index_context_set, contextualized_node):
+        from jedi.evaluate import analysis
+        # TODO this context is probably not right.
+        analysis.add(
+            contextualized_node.context,
+            'type-error-not-subscriptable',
+            contextualized_node.node,
+            message="TypeError: '%s' object is not subscriptable" % self
+        )
+        return NO_CONTEXTS
+
+    def py__iter__(self, contextualized_node=None):
+        if contextualized_node is not None:
+            from jedi.evaluate import analysis
+            analysis.add(
+                contextualized_node.context,
+                'type-error-not-iterable',
+                contextualized_node.node,
+                message="TypeError: '%s' object is not iterable" % self)
+        return iter([])
+
+    def get_signatures(self):
+        return []
+
+    def is_class(self):
+        return False
+
+    def is_instance(self):
+        return False
+
+    def is_function(self):
+        return False
+
+    def is_module(self):
+        return False
+
+    def is_namespace(self):
+        return False
+
+    def is_compiled(self):
+        return False
+
+    def is_bound_method(self):
+        return False
+
+    def py__bool__(self):
+        """
+        Since Wrapper is a super class for classes, functions and modules,
+        the return value will always be true.
+        """
+        return True
+
+    def py__doc__(self):
+        try:
+            self.tree_node.get_doc_node
+        except AttributeError:
+            return ''
+        else:
+            return clean_scope_docstring(self.tree_node)
+        return None
+
+    def get_safe_value(self, default=_sentinel):
+        if default is _sentinel:
+            raise ValueError("There exists no safe value for context %s" % self)
+        return default
+
+    def py__call__(self, arguments):
+        debug.warning("no execution possible %s", self)
+        return NO_CONTEXTS
+
+    def py__stop_iteration_returns(self):
+        debug.warning("Not possible to return the stop iterations of %s", self)
+        return NO_CONTEXTS
+
+    def get_qualified_names(self):
+        # Returns Optional[Tuple[str, ...]]
+        return None
+
+    def is_stub(self):
+        # The root context knows if it's a stub or not.
+        return self.parent_context.is_stub()
+
+
+def iterate_contexts(contexts, contextualized_node=None, is_async=False):
+    """
+    Calls `iterate`, on all contexts but ignores the ordering and just returns
+    all contexts that the iterate functions yield.
+    """
+    return ContextSet.from_sets(
+        lazy_context.infer()
+        for lazy_context in contexts.iterate(contextualized_node, is_async=is_async)
+    )
+
+
+class _ContextWrapperBase(HelperContextMixin):
+    predefined_names = {}
+
+    @safe_property
+    def name(self):
+        from jedi.evaluate.names import ContextName
+        wrapped_name = self._wrapped_context.name
+        if wrapped_name.tree_name is not None:
+            return ContextName(self, wrapped_name.tree_name)
+        else:
+            from jedi.evaluate.compiled import CompiledContextName
+            return CompiledContextName(self, wrapped_name.string_name)
+
+    @classmethod
+    @evaluator_as_method_param_cache()
+    def create_cached(cls, evaluator, *args, **kwargs):
+        return cls(*args, **kwargs)
+
+    def __getattr__(self, name):
+        assert name != '_wrapped_context', 'Problem with _get_wrapped_context'
+        return getattr(self._wrapped_context, name)
+
+
+class LazyContextWrapper(_ContextWrapperBase):
+    @safe_property
+    @memoize_method
+    def _wrapped_context(self):
+        with debug.increase_indent_cm('Resolve lazy context wrapper'):
+            return self._get_wrapped_context()
+
+    def __repr__(self):
+        return '<%s>' % (self.__class__.__name__)
+
+    def _get_wrapped_context(self):
+        raise NotImplementedError
+
+
+class ContextWrapper(_ContextWrapperBase):
+    def __init__(self, wrapped_context):
+        self._wrapped_context = wrapped_context
+
+    def __repr__(self):
+        return '%s(%s)' % (self.__class__.__name__, self._wrapped_context)
+
+
+class TreeContext(Context):
+    def __init__(self, evaluator, parent_context, tree_node):
+        super(TreeContext, self).__init__(evaluator, parent_context)
+        self.predefined_names = {}
+        self.tree_node = tree_node
+
+    def __repr__(self):
+        return '<%s: %s>' % (self.__class__.__name__, self.tree_node)
+
+
+class ContextualizedNode(object):
+    def __init__(self, context, node):
+        self.context = context
+        self.node = node
+
+    def get_root_context(self):
+        return self.context.get_root_context()
+
+    def infer(self):
+        return self.context.eval_node(self.node)
+
+    def __repr__(self):
+        return '<%s: %s in %s>' % (self.__class__.__name__, self.node, self.context)
+
+
+class ContextualizedName(ContextualizedNode):
+    # TODO merge with TreeNameDefinition?!
+    @property
+    def name(self):
+        return self.node
+
+    def assignment_indexes(self):
+        """
+        Returns an array of tuple(int, node) of the indexes that are used in
+        tuple assignments.
+
+        For example if the name is ``y`` in the following code::
+
+            x, (y, z) = 2, ''
+
+        would result in ``[(1, xyz_node), (0, yz_node)]``.
+
+        When searching for b in the case ``a, *b, c = [...]`` it will return::
+
+            [(slice(1, -1), abc_node)]
+        """
+        indexes = []
+        is_star_expr = False
+        node = self.node.parent
+        compare = self.node
+        while node is not None:
+            if node.type in ('testlist', 'testlist_comp', 'testlist_star_expr', 'exprlist'):
+                for i, child in enumerate(node.children):
+                    if child == compare:
+                        index = int(i / 2)
+                        if is_star_expr:
+                            from_end = int((len(node.children) - i) / 2)
+                            index = slice(index, -from_end)
+                        indexes.insert(0, (index, node))
+                        break
+                else:
+                    raise LookupError("Couldn't find the assignment.")
+                is_star_expr = False
+            elif node.type == 'star_expr':
+                is_star_expr = True
+            elif isinstance(node, (ExprStmt, SyncCompFor)):
+                break
+
+            compare = node
+            node = node.parent
+        return indexes
+
+
+def _getitem(context, index_contexts, contextualized_node):
+    from jedi.evaluate.context.iterable import Slice
+
+    # The actual getitem call.
+    simple_getitem = getattr(context, 'py__simple_getitem__', None)
+
+    result = NO_CONTEXTS
+    unused_contexts = set()
+    for index_context in index_contexts:
+        if simple_getitem is not None:
+            index = index_context
+            if isinstance(index_context, Slice):
+                index = index.obj
+
+            try:
+                method = index.get_safe_value
+            except AttributeError:
+                pass
+            else:
+                index = method(default=None)
+
+            if type(index) in (float, int, str, unicode, slice, bytes):
+                try:
+                    result |= simple_getitem(index)
+                    continue
+                except SimpleGetItemNotFound:
+                    pass
+
+        unused_contexts.add(index_context)
+
+    # The index was somehow not good enough or simply a wrong type.
+    # Therefore we now iterate through all the contexts and just take
+    # all results.
+    if unused_contexts or not index_contexts:
+        result |= context.py__getitem__(
+            ContextSet(unused_contexts),
+            contextualized_node
+        )
+    debug.dbg('py__getitem__ result: %s', result)
+    return result
+
+
+class ContextSet(BaseContextSet):
+    def py__class__(self):
+        return ContextSet(c.py__class__() for c in self._set)
+
+    def iterate(self, contextualized_node=None, is_async=False):
+        from jedi.evaluate.lazy_context import get_merged_lazy_context
+        type_iters = [c.iterate(contextualized_node, is_async=is_async) for c in self._set]
+        for lazy_contexts in zip_longest(*type_iters):
+            yield get_merged_lazy_context(
+                [l for l in lazy_contexts if l is not None]
+            )
+
+    def execute(self, arguments):
+        return ContextSet.from_sets(c.evaluator.execute(c, arguments) for c in self._set)
+
+    def execute_evaluated(self, *args, **kwargs):
+        return ContextSet.from_sets(c.execute_evaluated(*args, **kwargs) for c in self._set)
+
+    def py__getattribute__(self, *args, **kwargs):
+        if kwargs.get('is_goto'):
+            return reduce(add, [c.py__getattribute__(*args, **kwargs) for c in self._set], [])
+        return ContextSet.from_sets(c.py__getattribute__(*args, **kwargs) for c in self._set)
+
+    def get_item(self, *args, **kwargs):
+        return ContextSet.from_sets(_getitem(c, *args, **kwargs) for c in self._set)
+
+    def try_merge(self, function_name):
+        context_set = self.__class__([])
+        for c in self._set:
+            try:
+                method = getattr(c, function_name)
+            except AttributeError:
+                pass
+            else:
+                context_set |= method()
+        return context_set
+
+    def gather_annotation_classes(self):
+        return ContextSet.from_sets([c.gather_annotation_classes() for c in self._set])
+
+    def get_signatures(self):
+        return [sig for c in self._set for sig in c.get_signatures()]
+
+
+NO_CONTEXTS = ContextSet([])
+
+
+def iterator_to_context_set(func):
+    def wrapper(*args, **kwargs):
+        return ContextSet(func(*args, **kwargs))
+
+    return wrapper

jedi/evaluate/cache.py

@@ -1,15 +1,16 @@
 """
-- the popular ``memoize_default`` works like a typical memoize and returns the
+- the popular ``_memoize_default`` works like a typical memoize and returns the
   default otherwise.
-- ``CachedMetaClass`` uses ``memoize_default`` to do the same with classes.
+- ``CachedMetaClass`` uses ``_memoize_default`` to do the same with classes.
 """
 
-import inspect
+from jedi import debug
 
-NO_DEFAULT = object()
+_NO_DEFAULT = object()
+_RECURSION_SENTINEL = object()
 
 
-def memoize_default(default=NO_DEFAULT, evaluator_is_first_arg=False, second_arg_is_evaluator=False):
+def _memoize_default(default=_NO_DEFAULT, evaluator_is_first_arg=False, second_arg_is_evaluator=False):
     """ This is a typical memoization decorator, BUT there is one difference:
     To prevent recursion it sets defaults.
 
@@ -19,40 +20,104 @@ def memoize_default(default=NO_DEFAULT, evaluator_is_first_arg=False, second_arg
     """
     def func(function):
         def wrapper(obj, *args, **kwargs):
+            # TODO These checks are kind of ugly and slow.
             if evaluator_is_first_arg:
                 cache = obj.memoize_cache
-            elif second_arg_is_evaluator:  # needed for meta classes
-                cache = args[0].memoize_cache
+            elif second_arg_is_evaluator:
+                cache = args[0].memoize_cache  # needed for meta classes
             else:
                 cache = obj.evaluator.memoize_cache
 
             try:
                 memo = cache[function]
             except KeyError:
-                memo = {}
-                cache[function] = memo
+                cache[function] = memo = {}
 
             key = (obj, args, frozenset(kwargs.items()))
             if key in memo:
                 return memo[key]
             else:
-                if default is not NO_DEFAULT:
+                if default is not _NO_DEFAULT:
                     memo[key] = default
                 rv = function(obj, *args, **kwargs)
-                if inspect.isgenerator(rv):
-                    rv = list(rv)
                 memo[key] = rv
                 return rv
         return wrapper
+
     return func
 
 
+def evaluator_function_cache(default=_NO_DEFAULT):
+    def decorator(func):
+        return _memoize_default(default=default, evaluator_is_first_arg=True)(func)
+
+    return decorator
+
+
+def evaluator_method_cache(default=_NO_DEFAULT):
+    def decorator(func):
+        return _memoize_default(default=default)(func)
+
+    return decorator
+
+
+def evaluator_as_method_param_cache():
+    def decorator(call):
+        return _memoize_default(second_arg_is_evaluator=True)(call)
+
+    return decorator
+
+
 class CachedMetaClass(type):
     """
     This is basically almost the same than the decorator above, it just caches
     class initializations. Either you do it this way or with decorators, but
     with decorators you lose class access (isinstance, etc).
     """
-    @memoize_default(None, second_arg_is_evaluator=True)
+    @evaluator_as_method_param_cache()
     def __call__(self, *args, **kwargs):
         return super(CachedMetaClass, self).__call__(*args, **kwargs)
+
+
+def evaluator_method_generator_cache():
+    """
+    This is a special memoizer. It memoizes generators and also checks for
+    recursion errors and returns no further iterator elemends in that case.
+    """
+    def func(function):
+        def wrapper(obj, *args, **kwargs):
+            cache = obj.evaluator.memoize_cache
+            try:
+                memo = cache[function]
+            except KeyError:
+                cache[function] = memo = {}
+
+            key = (obj, args, frozenset(kwargs.items()))
+
+            if key in memo:
+                actual_generator, cached_lst = memo[key]
+            else:
+                actual_generator = function(obj, *args, **kwargs)
+                cached_lst = []
+                memo[key] = actual_generator, cached_lst
+
+            i = 0
+            while True:
+                try:
+                    next_element = cached_lst[i]
+                    if next_element is _RECURSION_SENTINEL:
+                        debug.warning('Found a generator recursion for %s' % obj)
+                        # This means we have hit a recursion.
+                        return
+                except IndexError:
+                    cached_lst.append(_RECURSION_SENTINEL)
+                    next_element = next(actual_generator, None)
+                    if next_element is None:
+                        cached_lst.pop()
+                        return
+                    cached_lst[-1] = next_element
+                yield next_element
+                i += 1
+        return wrapper
+
+    return func

jedi/evaluate/compiled/__init__.py

@@ -1,589 +1,64 @@
-"""
-Imitate the parser representation.
-"""
-import inspect
-import re
-import sys
-import os
-from functools import partial
-
-from jedi._compatibility import builtins as _builtins, unicode
-from jedi import debug
-from jedi.cache import underscore_memoization, memoize_method
-from jedi.parser.tree import Param, Operator
-from jedi.evaluate.helpers import FakeName
-from jedi.evaluate.filters import AbstractFilter, AbstractNameDefinition, \
-    ContextNameMixin
-from jedi.evaluate.context import Context, LazyKnownContext
-from . import fake
-
-
-_sep = os.path.sep
-if os.path.altsep is not None:
-    _sep += os.path.altsep
-_path_re = re.compile('(?:\.[^{0}]+|[{0}]__init__\.py)$'.format(re.escape(_sep)))
-del _sep
-
-
-class CheckAttribute(object):
-    """Raises an AttributeError if the attribute X isn't available."""
-    def __init__(self, func):
-        self.func = func
-        # Remove the py in front of e.g. py__call__.
-        self.check_name = func.__name__[2:]
-
-    def __get__(self, instance, owner):
-        # This might raise an AttributeError. That's wanted.
-        if self.check_name == '__iter__':
-            # Python iterators are a bit strange, because there's no need for
-            # the __iter__ function as long as __getitem__ is defined (it will
-            # just start with __getitem__(0). This is especially true for
-            # Python 2 strings, where `str.__iter__` is not even defined.
-            try:
-                iter(instance.obj)
-            except TypeError:
-                raise AttributeError
-        else:
-            getattr(instance.obj, self.check_name)
-        return partial(self.func, instance)
-
-
-class CompiledObject(Context):
-    path = None  # modules have this attribute - set it to None.
-    used_names = {}  # To be consistent with modules.
-
-    def __init__(self, evaluator, obj, parent_context=None, faked_class=None):
-        super(CompiledObject, self).__init__(evaluator, parent_context)
-        self.obj = obj
-        # This attribute will not be set for most classes, except for fakes.
-        self.tree_node = faked_class
-
-    def get_root_node(self):
-        # To make things a bit easier with filters we add this method here.
-        return self.get_root_context()
-
-    @CheckAttribute
-    def py__call__(self, params):
-        if inspect.isclass(self.obj):
-            from jedi.evaluate.instance import CompiledInstance
-            return set([CompiledInstance(self.evaluator, self.parent_context, self, params)])
-        else:
-            return set(self._execute_function(params))
-
-    @CheckAttribute
-    def py__class__(self):
-        return create(self.evaluator, self.obj.__class__)
-
-    @CheckAttribute
-    def py__mro__(self):
-        return (self,) + tuple(create(self.evaluator, cls) for cls in self.obj.__mro__[1:])
-
-    @CheckAttribute
-    def py__bases__(self):
-        return tuple(create(self.evaluator, cls) for cls in self.obj.__bases__)
-
-    def py__bool__(self):
-        return bool(self.obj)
-
-    def py__file__(self):
-        try:
-            return self.obj.__file__
-        except AttributeError:
-            return None
-
-    def is_class(self):
-        return inspect.isclass(self.obj)
-
-    @property
-    def doc(self):
-        return inspect.getdoc(self.obj) or ''
-
-    @property
-    def get_params(self):
-        return []  # TODO Fix me.
-        params_str, ret = self._parse_function_doc()
-        tokens = params_str.split(',')
-        if inspect.ismethoddescriptor(self.obj):
-            tokens.insert(0, 'self')
-        params = []
-        for p in tokens:
-            parts = [FakeName(part) for part in p.strip().split('=')]
-            if len(parts) > 1:
-                parts.insert(1, Operator('=', (0, 0)))
-            params.append(Param(parts, self))
-        return params
-
-    def get_param_names(self):
-        params_str, ret = self._parse_function_doc()
-        tokens = params_str.split(',')
-        if inspect.ismethoddescriptor(self.obj):
-            tokens.insert(0, 'self')
-        for p in tokens:
-            parts = p.strip().split('=')
-            if len(parts) > 1:
-                parts.insert(1, Operator('=', (0, 0)))
-            yield UnresolvableParamName(self, parts[0])
-
-    def __repr__(self):
-        return '<%s: %s>' % (self.__class__.__name__, repr(self.obj))
-
-    @underscore_memoization
-    def _parse_function_doc(self):
-        if self.doc is None:
-            return '', ''
-
-        return _parse_function_doc(self.doc)
-
-    @property
-    def api_type(self):
-        obj = self.obj
-        if inspect.isclass(obj):
-            return 'class'
-        elif inspect.ismodule(obj):
-            return 'module'
-        elif inspect.isbuiltin(obj) or inspect.ismethod(obj) \
-                or inspect.ismethoddescriptor(obj) or inspect.isfunction(obj):
-            return 'function'
-        # Everything else...
-        return 'instance'
-
-    @property
-    def type(self):
-        """Imitate the tree.Node.type values."""
-        cls = self._get_class()
-        if inspect.isclass(cls):
-            return 'classdef'
-        elif inspect.ismodule(cls):
-            return 'file_input'
-        elif inspect.isbuiltin(cls) or inspect.ismethod(cls) or \
-                inspect.ismethoddescriptor(cls):
-            return 'funcdef'
-
-    @underscore_memoization
-    def _cls(self):
-        """
-        We used to limit the lookups for instantiated objects like list(), but
-        this is not the case anymore. Python itself
-        """
-        # Ensures that a CompiledObject is returned that is not an instance (like list)
-        return self
-
-    def _get_class(self):
-        if not fake.is_class_instance(self.obj) or \
-                inspect.ismethoddescriptor(self.obj):  # slots
-            return self.obj
-
-        try:
-            return self.obj.__class__
-        except AttributeError:
-            # happens with numpy.core.umath._UFUNC_API (you get it
-            # automatically by doing `import numpy`.
-            return type
-
-    def get_filters(self, search_global=False, is_instance=False,
-                    until_position=None, origin_scope=None):
-        yield self._ensure_one_filter(is_instance)
-
-    @memoize_method
-    def _ensure_one_filter(self, is_instance):
-        """
-        search_global shouldn't change the fact that there's one dict, this way
-        there's only one `object`.
-        """
-        return CompiledObjectFilter(self.evaluator, self, is_instance)
-
-    def get_subscope_by_name(self, name):
-        if name in dir(self.obj):
-            return CompiledName(self.evaluator, self, name).parent
-        else:
-            raise KeyError("CompiledObject doesn't have an attribute '%s'." % name)
-
-    @CheckAttribute
-    def py__getitem__(self, index):
-        if type(self.obj) not in (str, list, tuple, unicode, bytes, bytearray, dict):
-            # Get rid of side effects, we won't call custom `__getitem__`s.
-            return set()
-
-        return set([create(self.evaluator, self.obj[index])])
-
-    @CheckAttribute
-    def py__iter__(self):
-        if type(self.obj) not in (str, list, tuple, unicode, bytes, bytearray, dict):
-            # Get rid of side effects, we won't call custom `__getitem__`s.
-            return
-
-        for part in self.obj:
-            yield LazyKnownContext(create(self.evaluator, part))
-
-    def py__name__(self):
-        try:
-            return self._get_class().__name__
-        except AttributeError:
-            return None
-
-    @property
-    def name(self):
-        try:
-            name = self._get_class().__name__
-        except AttributeError:
-            name = repr(self.obj)
-        return CompiledContextName(self, name)
-
-    def _execute_function(self, params):
-        if self.type != 'funcdef':
-            return
-
-        for name in self._parse_function_doc()[1].split():
-            try:
-                bltn_obj = getattr(_builtins, name)
-            except AttributeError:
-                continue
-            else:
-                if bltn_obj is None:
-                    # We want to evaluate everything except None.
-                    # TODO do we?
-                    continue
-                bltn_obj = create(self.evaluator, bltn_obj)
-                for result in self.evaluator.execute(bltn_obj, params):
-                    yield result
-
-    def is_scope(self):
-        return True
-
-    def get_self_attributes(self):
-        return []  # Instance compatibility
-
-    def get_imports(self):
-        return []  # Builtins don't have imports
-
-
-class CompiledName(AbstractNameDefinition):
-    def __init__(self, evaluator, parent_context, name):
-        self._evaluator = evaluator
-        self.parent_context = parent_context
-        self.string_name = name
-
-    def __repr__(self):
-        try:
-            name = self.parent_context.name  # __name__ is not defined all the time
-        except AttributeError:
-            name = None
-        return '<%s: (%s).%s>' % (self.__class__.__name__, name, self.string_name)
-
-    @property
-    def api_type(self):
-        return next(iter(self.infer())).api_type
-
-    @underscore_memoization
-    def infer(self):
-        module = self.parent_context.get_root_context()
-        return [_create_from_name(self._evaluator, module, self.parent_context, self.string_name)]
-
-
-class UnresolvableParamName(AbstractNameDefinition):
-    api_type = 'param'
-
-    def __init__(self, compiled_obj, name):
-        self.parent_context = compiled_obj.parent_context
-        self.string_name = name
-
-    def infer(self):
-        return set()
-
-
-class CompiledContextName(ContextNameMixin, AbstractNameDefinition):
-    def __init__(self, context, name):
-        self.string_name = name
-        self._context = context
-        self.parent_context = context.parent_context
-
-
-class EmptyCompiledName(AbstractNameDefinition):
-    """
-    Accessing some names will raise an exception. To avoid not having any
-    completions, just give Jedi the option to return this object. It infers to
-    nothing.
-    """
-    def __init__(self, evaluator, name):
-        self.parent_context = evaluator.BUILTINS
-        self.string_name = name
-
-    def infer(self):
-        return []
-
-
-class CompiledObjectFilter(AbstractFilter):
-    name_class = CompiledName
-
-    def __init__(self, evaluator, compiled_object, is_instance=False):
-        self._evaluator = evaluator
-        self._compiled_object = compiled_object
-        self._is_instance = is_instance
-
-    @memoize_method
-    def get(self, name):
-        name = str(name)
-        obj = self._compiled_object.obj
-        try:
-            getattr(obj, name)
-            if self._is_instance and name not in dir(obj):
-                return []
-        except AttributeError:
-            return []
-        except Exception:
-            # This is a bit ugly. We're basically returning this to make
-            # lookups possible without having the actual attribute. However
-            # this makes proper completion possible.
-            return [EmptyCompiledName(self._evaluator, name)]
-        return [self._create_name(name)]
-
-    def values(self):
-        obj = self._compiled_object.obj
-
-        names = []
-        for name in dir(obj):
-            names += self.get(name)
-
-        is_instance = self._is_instance or fake.is_class_instance(obj)
-        # ``dir`` doesn't include the type names.
-        if not inspect.ismodule(obj) and obj != type and not is_instance:
-            for filter in create(self._evaluator, type).get_filters():
-                names += filter.values()
-        return names
-
-    def _create_name(self, name):
-        return self.name_class(self._evaluator, self._compiled_object, name)
-
-
-def dotted_from_fs_path(fs_path, sys_path):
-    """
-    Changes `/usr/lib/python3.4/email/utils.py` to `email.utils`.  I.e.
-    compares the path with sys.path and then returns the dotted_path. If the
-    path is not in the sys.path, just returns None.
-    """
-    if os.path.basename(fs_path).startswith('__init__.'):
-        # We are calculating the path. __init__ files are not interesting.
-        fs_path = os.path.dirname(fs_path)
-
-    # prefer
-    #   - UNIX
-    #     /path/to/pythonX.Y/lib-dynload
-    #     /path/to/pythonX.Y/site-packages
-    #   - Windows
-    #     C:\path\to\DLLs
-    #     C:\path\to\Lib\site-packages
-    # over
-    #   - UNIX
-    #     /path/to/pythonX.Y
-    #   - Windows
-    #     C:\path\to\Lib
-    path = ''
-    for s in sys_path:
-        if (fs_path.startswith(s) and len(path) < len(s)):
-            path = s
-
-    # - Window
-    # X:\path\to\lib-dynload/datetime.pyd => datetime
-    module_path = fs_path[len(path):].lstrip(os.path.sep).lstrip('/')
-    # - Window
-    # Replace like X:\path\to\something/foo/bar.py
-    return _path_re.sub('', module_path).replace(os.path.sep, '.').replace('/', '.')
-
-
-def load_module(evaluator, path=None, name=None):
-    sys_path = evaluator.sys_path
-    if path is not None:
-        dotted_path = dotted_from_fs_path(path, sys_path=sys_path)
-    else:
-        dotted_path = name
-
-    if dotted_path is None:
-        p, _, dotted_path = path.partition(os.path.sep)
-        sys_path.insert(0, p)
-
-    temp, sys.path = sys.path, sys_path
-    try:
-        __import__(dotted_path)
-    except RuntimeError:
-        if 'PySide' in dotted_path or 'PyQt' in dotted_path:
-            # RuntimeError: the PyQt4.QtCore and PyQt5.QtCore modules both wrap
-            # the QObject class.
-            # See https://github.com/davidhalter/jedi/pull/483
-            return None
-        raise
-    except ImportError:
-        # If a module is "corrupt" or not really a Python module or whatever.
-        debug.warning('Module %s not importable.', path)
-        return None
-    finally:
-        sys.path = temp
-
-    # Just access the cache after import, because of #59 as well as the very
-    # complicated import structure of Python.
-    module = sys.modules[dotted_path]
-
-    return create(evaluator, module)
-
-
-docstr_defaults = {
-    'floating point number': 'float',
-    'character': 'str',
-    'integer': 'int',
-    'dictionary': 'dict',
-    'string': 'str',
-}
-
-
-def _parse_function_doc(doc):
-    """
-    Takes a function and returns the params and return value as a tuple.
-    This is nothing more than a docstring parser.
-
-    TODO docstrings like utime(path, (atime, mtime)) and a(b [, b]) -> None
-    TODO docstrings like 'tuple of integers'
-    """
-    # parse round parentheses: def func(a, (b,c))
-    try:
-        count = 0
-        start = doc.index('(')
-        for i, s in enumerate(doc[start:]):
-            if s == '(':
-                count += 1
-            elif s == ')':
-                count -= 1
-            if count == 0:
-                end = start + i
-                break
-        param_str = doc[start + 1:end]
-    except (ValueError, UnboundLocalError):
-        # ValueError for doc.index
-        # UnboundLocalError for undefined end in last line
-        debug.dbg('no brackets found - no param')
-        end = 0
-        param_str = ''
-    else:
-        # remove square brackets, that show an optional param ( = None)
-        def change_options(m):
-            args = m.group(1).split(',')
-            for i, a in enumerate(args):
-                if a and '=' not in a:
-                    args[i] += '=None'
-            return ','.join(args)
-
-        while True:
-            param_str, changes = re.subn(r' ?\[([^\[\]]+)\]',
-                                         change_options, param_str)
-            if changes == 0:
-                break
-    param_str = param_str.replace('-', '_')  # see: isinstance.__doc__
-
-    # parse return value
-    r = re.search('-[>-]* ', doc[end:end + 7])
-    if r is None:
-        ret = ''
-    else:
-        index = end + r.end()
-        # get result type, which can contain newlines
-        pattern = re.compile(r'(,\n|[^\n-])+')
-        ret_str = pattern.match(doc, index).group(0).strip()
-        # New object -> object()
-        ret_str = re.sub(r'[nN]ew (.*)', r'\1()', ret_str)
-
-        ret = docstr_defaults.get(ret_str, ret_str)
-
-    return param_str, ret
-
-
-def _create_from_name(evaluator, module, compiled_object, name):
-    obj = compiled_object.obj
-    faked = None
-    try:
-        faked = fake.get_faked(evaluator, module, obj, parent_context=compiled_object, name=name)
-        if faked.type == 'funcdef':
-            from jedi.evaluate.representation import FunctionContext
-            return FunctionContext(evaluator, compiled_object, faked)
-    except fake.FakeDoesNotExist:
-        pass
-
-    try:
-        obj = getattr(obj, name)
-    except AttributeError:
-        # Happens e.g. in properties of
-        # PyQt4.QtGui.QStyleOptionComboBox.currentText
-        # -> just set it to None
-        obj = None
-    return create(evaluator, obj, parent_context=compiled_object, faked=faked)
+from jedi._compatibility import unicode
+from jedi.evaluate.compiled.context import CompiledObject, CompiledName, \
+    CompiledObjectFilter, CompiledContextName, create_from_access_path
+from jedi.evaluate.base_context import ContextWrapper, LazyContextWrapper
 
 
 def builtin_from_name(evaluator, string):
-    bltn_obj = getattr(_builtins, string)
-    return create(evaluator, bltn_obj)
-
-
-def _a_generator(foo):
-    """Used to have an object to return for generators."""
-    yield 42
-    yield foo
-
-
-_SPECIAL_OBJECTS = {
-    'FUNCTION_CLASS': type(load_module),
-    'METHOD_CLASS': type(CompiledObject.is_class),
-    'MODULE_CLASS': type(os),
-    'GENERATOR_OBJECT': _a_generator(1.0),
-    'BUILTINS': _builtins,
-}
-
-
-def get_special_object(evaluator, identifier):
-    obj = _SPECIAL_OBJECTS[identifier]
-    return create(evaluator, obj, parent_context=create(evaluator, _builtins))
-
-
-def compiled_objects_cache(attribute_name):
-    def decorator(func):
-        """
-        This decorator caches just the ids, oopposed to caching the object itself.
-        Caching the id has the advantage that an object doesn't need to be
-        hashable.
-        """
-        def wrapper(evaluator, obj, parent_context=None, module=None, faked=None):
-            cache = getattr(evaluator, attribute_name)
-            # Do a very cheap form of caching here.
-            key = id(obj), id(parent_context)
-            try:
-                return cache[key][0]
-            except KeyError:
-                # TODO this whole decorator is way too ugly
-                result = func(evaluator, obj, parent_context, module, faked)
-                # Need to cache all of them, otherwise the id could be overwritten.
-                cache[key] = result, obj, parent_context, module, faked
-                return result
-        return wrapper
-
-    return decorator
-
-
-@compiled_objects_cache('compiled_cache')
-def create(evaluator, obj, parent_context=None, module=None, faked=None):
-    """
-    A very weird interface class to this module. The more options provided the
-    more acurate loading compiled objects is.
-    """
-    if inspect.ismodule(obj):
-        if parent_context is not None:
-            # Modules don't have parents, be careful with caching: recurse.
-            return create(evaluator, obj)
+    typing_builtins_module = evaluator.builtins_module
+    if string in ('None', 'True', 'False'):
+        builtins, = typing_builtins_module.non_stub_context_set
+        filter_ = next(builtins.get_filters())
     else:
-        if parent_context is None and obj != _builtins:
-            return create(evaluator, obj, create(evaluator, _builtins))
+        filter_ = next(typing_builtins_module.get_filters())
+    name, = filter_.get(string)
+    context, = name.infer()
+    return context
 
-        try:
-            faked = fake.get_faked(evaluator, module, obj, parent_context=parent_context)
-            if faked.type == 'funcdef':
-                from jedi.evaluate.representation import FunctionContext
-                return FunctionContext(evaluator, parent_context, faked)
-        except fake.FakeDoesNotExist:
-            pass
 
-    return CompiledObject(evaluator, obj, parent_context, faked)
+class CompiledValue(LazyContextWrapper):
+    def __init__(self, compiled_obj):
+        self.evaluator = compiled_obj.evaluator
+        self._compiled_obj = compiled_obj
+
+    def __getattribute__(self, name):
+        if name in ('get_safe_value', 'execute_operation', 'access_handle',
+                    'negate', 'py__bool__', 'is_compiled'):
+            return getattr(self._compiled_obj, name)
+        return super(CompiledValue, self).__getattribute__(name)
+
+    def _get_wrapped_context(self):
+        instance, = builtin_from_name(
+            self.evaluator, self._compiled_obj.name.string_name).execute_evaluated()
+        return instance
+
+    def __repr__(self):
+        return '<%s: %s>' % (self.__class__.__name__, self._compiled_obj)
+
+
+def create_simple_object(evaluator, obj):
+    """
+    Only allows creations of objects that are easily picklable across Python
+    versions.
+    """
+    assert type(obj) in (int, float, str, bytes, unicode, slice, complex, bool), obj
+    compiled_obj = create_from_access_path(
+        evaluator,
+        evaluator.compiled_subprocess.create_simple_object(obj)
+    )
+    return CompiledValue(compiled_obj)
+
+
+def get_string_context_set(evaluator):
+    return builtin_from_name(evaluator, u'str').execute_evaluated()
+
+
+def load_module(evaluator, dotted_name, **kwargs):
+    # Temporary, some tensorflow builtins cannot be loaded, so it's tried again
+    # and again and it's really slow.
+    if dotted_name.startswith('tensorflow.'):
+        return None
+    access_path = evaluator.compiled_subprocess.load_module(dotted_name=dotted_name, **kwargs)
+    if access_path is None:
+        return None
+    return create_from_access_path(evaluator, access_path)

jedi/evaluate/compiled/access.py

@@ -0,0 +1,497 @@
+from __future__ import print_function
+import inspect
+import types
+import sys
+import operator as op
+from collections import namedtuple
+
+from jedi._compatibility import unicode, is_py3, builtins, \
+    py_version, force_unicode
+from jedi.evaluate.compiled.getattr_static import getattr_static
+
+ALLOWED_GETITEM_TYPES = (str, list, tuple, unicode, bytes, bytearray, dict)
+
+MethodDescriptorType = type(str.replace)
+# These are not considered classes and access is granted even though they have
+# a __class__ attribute.
+NOT_CLASS_TYPES = (
+    types.BuiltinFunctionType,
+    types.CodeType,
+    types.FrameType,
+    types.FunctionType,
+    types.GeneratorType,
+    types.GetSetDescriptorType,
+    types.LambdaType,
+    types.MemberDescriptorType,
+    types.MethodType,
+    types.ModuleType,
+    types.TracebackType,
+    MethodDescriptorType
+)
+
+if is_py3:
+    NOT_CLASS_TYPES += (
+        types.MappingProxyType,
+        types.SimpleNamespace,
+        types.DynamicClassAttribute,
+    )
+
+
+# Those types don't exist in typing.
+MethodDescriptorType = type(str.replace)
+WrapperDescriptorType = type(set.__iter__)
+# `object.__subclasshook__` is an already executed descriptor.
+object_class_dict = type.__dict__["__dict__"].__get__(object)
+ClassMethodDescriptorType = type(object_class_dict['__subclasshook__'])
+
+_sentinel = object()
+
+# Maps Python syntax to the operator module.
+COMPARISON_OPERATORS = {
+    '==': op.eq,
+    '!=': op.ne,
+    'is': op.is_,
+    'is not': op.is_not,
+    '<': op.lt,
+    '<=': op.le,
+    '>': op.gt,
+    '>=': op.ge,
+}
+
+_OPERATORS = {
+    '+': op.add,
+    '-': op.sub,
+}
+_OPERATORS.update(COMPARISON_OPERATORS)
+
+ALLOWED_DESCRIPTOR_ACCESS = (
+    types.FunctionType,
+    types.GetSetDescriptorType,
+    types.MemberDescriptorType,
+    MethodDescriptorType,
+    WrapperDescriptorType,
+    ClassMethodDescriptorType,
+    staticmethod,
+    classmethod,
+)
+
+
+def safe_getattr(obj, name, default=_sentinel):
+    try:
+        attr, is_get_descriptor = getattr_static(obj, name)
+    except AttributeError:
+        if default is _sentinel:
+            raise
+        return default
+    else:
+        if isinstance(attr, ALLOWED_DESCRIPTOR_ACCESS):
+            # In case of descriptors that have get methods we cannot return
+            # it's value, because that would mean code execution.
+            # Since it's an isinstance call, code execution is still possible,
+            # but this is not really a security feature, but much more of a
+            # safety feature. Code execution is basically always possible when
+            # a module is imported. This is here so people don't shoot
+            # themselves in the foot.
+            return getattr(obj, name)
+    return attr
+
+
+SignatureParam = namedtuple(
+    'SignatureParam',
+    'name has_default default default_string has_annotation annotation annotation_string kind_name'
+)
+
+
+def compiled_objects_cache(attribute_name):
+    def decorator(func):
+        """
+        This decorator caches just the ids, oopposed to caching the object itself.
+        Caching the id has the advantage that an object doesn't need to be
+        hashable.
+        """
+        def wrapper(evaluator, obj, parent_context=None):
+            cache = getattr(evaluator, attribute_name)
+            # Do a very cheap form of caching here.
+            key = id(obj)
+            try:
+                cache[key]
+                return cache[key][0]
+            except KeyError:
+                # TODO wuaaaarrghhhhhhhh
+                if attribute_name == 'mixed_cache':
+                    result = func(evaluator, obj, parent_context)
+                else:
+                    result = func(evaluator, obj)
+                # Need to cache all of them, otherwise the id could be overwritten.
+                cache[key] = result, obj, parent_context
+                return result
+        return wrapper
+
+    return decorator
+
+
+def create_access(evaluator, obj):
+    return evaluator.compiled_subprocess.get_or_create_access_handle(obj)
+
+
+def load_module(evaluator, dotted_name, sys_path):
+    temp, sys.path = sys.path, sys_path
+    try:
+        __import__(dotted_name)
+    except ImportError:
+        # If a module is "corrupt" or not really a Python module or whatever.
+        print('Module %s not importable in path %s.' % (dotted_name, sys_path), file=sys.stderr)
+        return None
+    except Exception:
+        # Since __import__ pretty much makes code execution possible, just
+        # catch any error here and print it.
+        import traceback
+        print("Cannot import:\n%s" % traceback.format_exc(), file=sys.stderr)
+        return None
+    finally:
+        sys.path = temp
+
+    # Just access the cache after import, because of #59 as well as the very
+    # complicated import structure of Python.
+    module = sys.modules[dotted_name]
+    return create_access_path(evaluator, module)
+
+
+class AccessPath(object):
+    def __init__(self, accesses):
+        self.accesses = accesses
+
+    # Writing both of these methods here looks a bit ridiculous. However with
+    # the differences of Python 2/3 it's actually necessary, because we will
+    # otherwise have a accesses attribute that is bytes instead of unicode.
+    def __getstate__(self):
+        return self.accesses
+
+    def __setstate__(self, value):
+        self.accesses = value
+
+
+def create_access_path(evaluator, obj):
+    access = create_access(evaluator, obj)
+    return AccessPath(access.get_access_path_tuples())
+
+
+def _force_unicode_decorator(func):
+    return lambda *args, **kwargs: force_unicode(func(*args, **kwargs))
+
+
+def get_api_type(obj):
+    if inspect.isclass(obj):
+        return u'class'
+    elif inspect.ismodule(obj):
+        return u'module'
+    elif inspect.isbuiltin(obj) or inspect.ismethod(obj) \
+            or inspect.ismethoddescriptor(obj) or inspect.isfunction(obj):
+        return u'function'
+    # Everything else...
+    return u'instance'
+
+
+class DirectObjectAccess(object):
+    def __init__(self, evaluator, obj):
+        self._evaluator = evaluator
+        self._obj = obj
+
+    def __repr__(self):
+        return '%s(%s)' % (self.__class__.__name__, self.get_repr())
+
+    def _create_access(self, obj):
+        return create_access(self._evaluator, obj)
+
+    def _create_access_path(self, obj):
+        return create_access_path(self._evaluator, obj)
+
+    def py__bool__(self):
+        return bool(self._obj)
+
+    def py__file__(self):
+        try:
+            return self._obj.__file__
+        except AttributeError:
+            return None
+
+    def py__doc__(self):
+        return force_unicode(inspect.getdoc(self._obj)) or u''
+
+    def py__name__(self):
+        if not _is_class_instance(self._obj) or \
+                inspect.ismethoddescriptor(self._obj):  # slots
+            cls = self._obj
+        else:
+            try:
+                cls = self._obj.__class__
+            except AttributeError:
+                # happens with numpy.core.umath._UFUNC_API (you get it
+                # automatically by doing `import numpy`.
+                return None
+
+        try:
+            return force_unicode(cls.__name__)
+        except AttributeError:
+            return None
+
+    def py__mro__accesses(self):
+        return tuple(self._create_access_path(cls) for cls in self._obj.__mro__[1:])
+
+    def py__getitem__all_values(self):
+        if isinstance(self._obj, dict):
+            return [self._create_access_path(v) for v in self._obj.values()]
+        return self.py__iter__list()
+
+    def py__simple_getitem__(self, index):
+        if type(self._obj) not in ALLOWED_GETITEM_TYPES:
+            # Get rid of side effects, we won't call custom `__getitem__`s.
+            return None
+
+        return self._create_access_path(self._obj[index])
+
+    def py__iter__list(self):
+        if not hasattr(self._obj, '__getitem__'):
+            return None
+
+        if type(self._obj) not in ALLOWED_GETITEM_TYPES:
+            # Get rid of side effects, we won't call custom `__getitem__`s.
+            return []
+
+        lst = []
+        for i, part in enumerate(self._obj):
+            if i > 20:
+                # Should not go crazy with large iterators
+                break
+            lst.append(self._create_access_path(part))
+        return lst
+
+    def py__class__(self):
+        return self._create_access_path(self._obj.__class__)
+
+    def py__bases__(self):
+        return [self._create_access_path(base) for base in self._obj.__bases__]
+
+    def py__path__(self):
+        return self._obj.__path__
+
+    @_force_unicode_decorator
+    def get_repr(self):
+        builtins = 'builtins', '__builtin__'
+
+        if inspect.ismodule(self._obj):
+            return repr(self._obj)
+        # Try to avoid execution of the property.
+        if safe_getattr(self._obj, '__module__', default='') in builtins:
+            return repr(self._obj)
+
+        type_ = type(self._obj)
+        if type_ == type:
+            return type.__repr__(self._obj)
+
+        if safe_getattr(type_, '__module__', default='') in builtins:
+            # Allow direct execution of repr for builtins.
+            return repr(self._obj)
+        return object.__repr__(self._obj)
+
+    def is_class(self):
+        return inspect.isclass(self._obj)
+
+    def is_module(self):
+        return inspect.ismodule(self._obj)
+
+    def is_instance(self):
+        return _is_class_instance(self._obj)
+
+    def ismethoddescriptor(self):
+        return inspect.ismethoddescriptor(self._obj)
+
+    def get_qualified_names(self):
+        def try_to_get_name(obj):
+            return getattr(obj, '__qualname__', getattr(obj, '__name__', None))
+
+        if self.is_module():
+            return ()
+        name = try_to_get_name(self._obj)
+        if name is None:
+            name = try_to_get_name(type(self._obj))
+            if name is None:
+                return ()
+        return tuple(name.split('.'))
+
+    def dir(self):
+        return list(map(force_unicode, dir(self._obj)))
+
+    def has_iter(self):
+        try:
+            iter(self._obj)
+            return True
+        except TypeError:
+            return False
+
+    def is_allowed_getattr(self, name):
+        # TODO this API is ugly.
+        try:
+            attr, is_get_descriptor = getattr_static(self._obj, name)
+        except AttributeError:
+            return False, False
+        else:
+            if is_get_descriptor and type(attr) not in ALLOWED_DESCRIPTOR_ACCESS:
+                # In case of descriptors that have get methods we cannot return
+                # it's value, because that would mean code execution.
+                return True, True
+        return True, False
+
+    def getattr_paths(self, name, default=_sentinel):
+        try:
+            return_obj = getattr(self._obj, name)
+        except Exception as e:
+            if default is _sentinel:
+                if isinstance(e, AttributeError):
+                    # Happens e.g. in properties of
+                    # PyQt4.QtGui.QStyleOptionComboBox.currentText
+                    # -> just set it to None
+                    raise
+                # Just in case anything happens, return an AttributeError. It
+                # should not crash.
+                raise AttributeError
+            return_obj = default
+        access = self._create_access(return_obj)
+        if inspect.ismodule(return_obj):
+            return [access]
+
+        module = inspect.getmodule(return_obj)
+        if module is None:
+            module = inspect.getmodule(type(return_obj))
+            if module is None:
+                module = builtins
+        return [self._create_access(module), access]
+
+    def get_safe_value(self):
+        if type(self._obj) in (bool, bytes, float, int, str, unicode, slice):
+            return self._obj
+        raise ValueError("Object is type %s and not simple" % type(self._obj))
+
+    def get_api_type(self):
+        return get_api_type(self._obj)
+
+    def get_access_path_tuples(self):
+        accesses = [create_access(self._evaluator, o) for o in self._get_objects_path()]
+        return [(access.py__name__(), access) for access in accesses]
+
+    def _get_objects_path(self):
+        def get():
+            obj = self._obj
+            yield obj
+            try:
+                obj = obj.__objclass__
+            except AttributeError:
+                pass
+            else:
+                yield obj
+
+            try:
+                # Returns a dotted string path.
+                imp_plz = obj.__module__
+            except AttributeError:
+                # Unfortunately in some cases like `int` there's no __module__
+                if not inspect.ismodule(obj):
+                    yield builtins
+            else:
+                if imp_plz is None:
+                    # Happens for example in `(_ for _ in []).send.__module__`.
+                    yield builtins
+                else:
+                    try:
+                        yield sys.modules[imp_plz]
+                    except KeyError:
+                        # __module__ can be something arbitrary that doesn't exist.
+                        yield builtins
+
+        return list(reversed(list(get())))
+
+    def execute_operation(self, other_access_handle, operator):
+        other_access = other_access_handle.access
+        op = _OPERATORS[operator]
+        return self._create_access_path(op(self._obj, other_access._obj))
+
+    def needs_type_completions(self):
+        return inspect.isclass(self._obj) and self._obj != type
+
+    def get_signature_params(self):
+        return [
+            SignatureParam(
+                name=p.name,
+                has_default=p.default is not p.empty,
+                default=self._create_access_path(p.default),
+                default_string=repr(p.default),
+                has_annotation=p.annotation is not p.empty,
+                annotation=self._create_access_path(p.annotation),
+                annotation_string=str(p.default),
+                kind_name=str(p.kind)
+            ) for p in self._get_signature().parameters.values()
+        ]
+
+    def _get_signature(self):
+        obj = self._obj
+        if py_version < 33:
+            raise ValueError("inspect.signature was introduced in 3.3")
+        if py_version == 34:
+            # In 3.4 inspect.signature are wrong for str and int. This has
+            # been fixed in 3.5. The signature of object is returned,
+            # because no signature was found for str. Here we imitate 3.5
+            # logic and just ignore the signature if the magic methods
+            # don't match object.
+            # 3.3 doesn't even have the logic and returns nothing for str
+            # and classes that inherit from object.
+            user_def = inspect._signature_get_user_defined_method
+            if (inspect.isclass(obj)
+                    and not user_def(type(obj), '__init__')
+                    and not user_def(type(obj), '__new__')
+                    and (obj.__init__ != object.__init__
+                         or obj.__new__ != object.__new__)):
+                raise ValueError
+
+        try:
+            return inspect.signature(obj)
+        except (RuntimeError, TypeError):
+            # Reading the code of the function in Python 3.6 implies there are
+            # at least these errors that might occur if something is wrong with
+            # the signature. In that case we just want a simple escape for now.
+            raise ValueError
+
+    def get_return_annotation(self):
+        try:
+            o = self._obj.__annotations__.get('return')
+        except AttributeError:
+            return None
+
+        if o is None:
+            return None
+
+        return self._create_access_path(o)
+
+    def negate(self):
+        return self._create_access_path(-self._obj)
+
+    def get_dir_infos(self):
+        """
+        Used to return a couple of infos that are needed when accessing the sub
+        objects of an objects
+        """
+        # TODO is_allowed_getattr might raise an AttributeError
+        tuples = dict(
+            (force_unicode(name), self.is_allowed_getattr(name))
+            for name in self.dir()
+        )
+        return self.needs_type_completions(), tuples
+
+
+def _is_class_instance(obj):
+    """Like inspect.* methods."""
+    try:
+        cls = obj.__class__
+    except AttributeError:
+        return False
+    else:
+        return cls != type and not issubclass(cls, NOT_CLASS_TYPES)

jedi/evaluate/compiled/context.py

@@ -0,0 +1,541 @@
+"""
+Imitate the parser representation.
+"""
+import re
+from functools import partial
+
+from jedi import debug
+from jedi.evaluate.utils import to_list
+from jedi._compatibility import force_unicode, Parameter, cast_path
+from jedi.cache import underscore_memoization, memoize_method
+from jedi.evaluate.filters import AbstractFilter
+from jedi.evaluate.names import AbstractNameDefinition, ContextNameMixin, \
+    ParamNameInterface
+from jedi.evaluate.base_context import Context, ContextSet, NO_CONTEXTS
+from jedi.evaluate.lazy_context import LazyKnownContext
+from jedi.evaluate.compiled.access import _sentinel
+from jedi.evaluate.cache import evaluator_function_cache
+from jedi.evaluate.helpers import reraise_getitem_errors
+from jedi.evaluate.signature import BuiltinSignature
+
+
+class CheckAttribute(object):
+    """Raises an AttributeError if the attribute X isn't available."""
+    def __init__(self, check_name=None):
+        # Remove the py in front of e.g. py__call__.
+        self.check_name = check_name
+
+    def __call__(self, func):
+        self.func = func
+        if self.check_name is None:
+            self.check_name = force_unicode(func.__name__[2:])
+        return self
+
+    def __get__(self, instance, owner):
+        if instance is None:
+            return self
+
+        # This might raise an AttributeError. That's wanted.
+        instance.access_handle.getattr_paths(self.check_name)
+        return partial(self.func, instance)
+
+
+class CompiledObject(Context):
+    def __init__(self, evaluator, access_handle, parent_context=None):
+        super(CompiledObject, self).__init__(evaluator, parent_context)
+        self.access_handle = access_handle
+
+    def py__call__(self, arguments):
+        return_annotation = self.access_handle.get_return_annotation()
+        if return_annotation is not None:
+            # TODO the return annotation may also be a string.
+            return create_from_access_path(self.evaluator, return_annotation).execute_annotation()
+
+        try:
+            self.access_handle.getattr_paths(u'__call__')
+        except AttributeError:
+            return super(CompiledObject, self).py__call__(arguments)
+        else:
+            if self.access_handle.is_class():
+                from jedi.evaluate.context import CompiledInstance
+                return ContextSet([
+                    CompiledInstance(self.evaluator, self.parent_context, self, arguments)
+                ])
+            else:
+                return ContextSet(self._execute_function(arguments))
+
+    @CheckAttribute()
+    def py__class__(self):
+        return create_from_access_path(self.evaluator, self.access_handle.py__class__())
+
+    @CheckAttribute()
+    def py__mro__(self):
+        return (self,) + tuple(
+            create_from_access_path(self.evaluator, access)
+            for access in self.access_handle.py__mro__accesses()
+        )
+
+    @CheckAttribute()
+    def py__bases__(self):
+        return tuple(
+            create_from_access_path(self.evaluator, access)
+            for access in self.access_handle.py__bases__()
+        )
+
+    @CheckAttribute()
+    def py__path__(self):
+        return map(cast_path, self.access_handle.py__path__())
+
+    @property
+    def string_names(self):
+        # For modules
+        name = self.py__name__()
+        if name is None:
+            return ()
+        return tuple(name.split('.'))
+
+    def get_qualified_names(self):
+        return self.access_handle.get_qualified_names()
+
+    def py__bool__(self):
+        return self.access_handle.py__bool__()
+
+    def py__file__(self):
+        return cast_path(self.access_handle.py__file__())
+
+    def is_class(self):
+        return self.access_handle.is_class()
+
+    def is_module(self):
+        return self.access_handle.is_module()
+
+    def is_compiled(self):
+        return True
+
+    def is_stub(self):
+        return False
+
+    def is_instance(self):
+        return self.access_handle.is_instance()
+
+    def py__doc__(self):
+        return self.access_handle.py__doc__()
+
+    @to_list
+    def get_param_names(self):
+        try:
+            signature_params = self.access_handle.get_signature_params()
+        except ValueError:  # Has no signature
+            params_str, ret = self._parse_function_doc()
+            if not params_str:
+                tokens = []
+            else:
+                tokens = params_str.split(',')
+            if self.access_handle.ismethoddescriptor():
+                tokens.insert(0, 'self')
+            for p in tokens:
+                name, _, default = p.strip().partition('=')
+                yield UnresolvableParamName(self, name, default)
+        else:
+            for signature_param in signature_params:
+                yield SignatureParamName(self, signature_param)
+
+    def get_signatures(self):
+        _, return_string = self._parse_function_doc()
+        return [BuiltinSignature(self, return_string)]
+
+    def __repr__(self):
+        return '<%s: %s>' % (self.__class__.__name__, self.access_handle.get_repr())
+
+    @underscore_memoization
+    def _parse_function_doc(self):
+        doc = self.py__doc__()
+        if doc is None:
+            return '', ''
+
+        return _parse_function_doc(doc)
+
+    @property
+    def api_type(self):
+        return self.access_handle.get_api_type()
+
+    @underscore_memoization
+    def _cls(self):
+        """
+        We used to limit the lookups for instantiated objects like list(), but
+        this is not the case anymore. Python itself
+        """
+        # Ensures that a CompiledObject is returned that is not an instance (like list)
+        return self
+
+    def get_filters(self, search_global=False, is_instance=False,
+                    until_position=None, origin_scope=None):
+        yield self._ensure_one_filter(is_instance)
+
+    @memoize_method
+    def _ensure_one_filter(self, is_instance):
+        """
+        search_global shouldn't change the fact that there's one dict, this way
+        there's only one `object`.
+        """
+        return CompiledObjectFilter(self.evaluator, self, is_instance)
+
+    @CheckAttribute(u'__getitem__')
+    def py__simple_getitem__(self, index):
+        with reraise_getitem_errors(IndexError, KeyError, TypeError):
+            access = self.access_handle.py__simple_getitem__(index)
+        if access is None:
+            return NO_CONTEXTS
+
+        return ContextSet([create_from_access_path(self.evaluator, access)])
+
+    def py__getitem__(self, index_context_set, contextualized_node):
+        all_access_paths = self.access_handle.py__getitem__all_values()
+        if all_access_paths is None:
+            # This means basically that no __getitem__ has been defined on this
+            # object.
+            return super(CompiledObject, self).py__getitem__(index_context_set, contextualized_node)
+        return ContextSet(
+            create_from_access_path(self.evaluator, access)
+            for access in all_access_paths
+        )
+
+    def py__iter__(self, contextualized_node=None):
+        # Python iterators are a bit strange, because there's no need for
+        # the __iter__ function as long as __getitem__ is defined (it will
+        # just start with __getitem__(0). This is especially true for
+        # Python 2 strings, where `str.__iter__` is not even defined.
+        if not self.access_handle.has_iter():
+            for x in super(CompiledObject, self).py__iter__(contextualized_node):
+                yield x
+
+        access_path_list = self.access_handle.py__iter__list()
+        if access_path_list is None:
+            # There is no __iter__ method on this object.
+            return
+
+        for access in access_path_list:
+            yield LazyKnownContext(create_from_access_path(self.evaluator, access))
+
+    def py__name__(self):
+        return self.access_handle.py__name__()
+
+    @property
+    def name(self):
+        name = self.py__name__()
+        if name is None:
+            name = self.access_handle.get_repr()
+        return CompiledContextName(self, name)
+
+    def _execute_function(self, params):
+        from jedi.evaluate import docstrings
+        from jedi.evaluate.compiled import builtin_from_name
+        if self.api_type != 'function':
+            return
+
+        for name in self._parse_function_doc()[1].split():
+            try:
+                # TODO wtf is this? this is exactly the same as the thing
+                # below. It uses getattr as well.
+                self.evaluator.builtins_module.access_handle.getattr_paths(name)
+            except AttributeError:
+                continue
+            else:
+                bltn_obj = builtin_from_name(self.evaluator, name)
+                for result in self.evaluator.execute(bltn_obj, params):
+                    yield result
+        for type_ in docstrings.infer_return_types(self):
+            yield type_
+
+    def get_safe_value(self, default=_sentinel):
+        try:
+            return self.access_handle.get_safe_value()
+        except ValueError:
+            if default == _sentinel:
+                raise
+            return default
+
+    def execute_operation(self, other, operator):
+        return create_from_access_path(
+            self.evaluator,
+            self.access_handle.execute_operation(other.access_handle, operator)
+        )
+
+    def negate(self):
+        return create_from_access_path(self.evaluator, self.access_handle.negate())
+
+    def get_metaclasses(self):
+        return NO_CONTEXTS
+
+
+class CompiledName(AbstractNameDefinition):
+    def __init__(self, evaluator, parent_context, name):
+        self._evaluator = evaluator
+        self.parent_context = parent_context
+        self.string_name = name
+
+    def _get_qualified_names(self):
+        parent_qualified_names = self.parent_context.get_qualified_names()
+        return parent_qualified_names + (self.string_name,)
+
+    def __repr__(self):
+        try:
+            name = self.parent_context.name  # __name__ is not defined all the time
+        except AttributeError:
+            name = None
+        return '<%s: (%s).%s>' % (self.__class__.__name__, name, self.string_name)
+
+    @property
+    def api_type(self):
+        api = self.infer()
+        # If we can't find the type, assume it is an instance variable
+        if not api:
+            return "instance"
+        return next(iter(api)).api_type
+
+    @underscore_memoization
+    def infer(self):
+        return ContextSet([_create_from_name(
+            self._evaluator, self.parent_context, self.string_name
+        )])
+
+
+class SignatureParamName(ParamNameInterface, AbstractNameDefinition):
+    def __init__(self, compiled_obj, signature_param):
+        self.parent_context = compiled_obj.parent_context
+        self._signature_param = signature_param
+
+    @property
+    def string_name(self):
+        return self._signature_param.name
+
+    def to_string(self):
+        s = self._kind_string() + self.string_name
+        if self._signature_param.has_annotation:
+            s += ': ' + self._signature_param.annotation_string
+        if self._signature_param.has_default:
+            s += '=' + self._signature_param.default_string
+        return s
+
+    def get_kind(self):
+        return getattr(Parameter, self._signature_param.kind_name)
+
+    def infer(self):
+        p = self._signature_param
+        evaluator = self.parent_context.evaluator
+        contexts = NO_CONTEXTS
+        if p.has_default:
+            contexts = ContextSet([create_from_access_path(evaluator, p.default)])
+        if p.has_annotation:
+            annotation = create_from_access_path(evaluator, p.annotation)
+            contexts |= annotation.execute_evaluated()
+        return contexts
+
+
+class UnresolvableParamName(ParamNameInterface, AbstractNameDefinition):
+    def __init__(self, compiled_obj, name, default):
+        self.parent_context = compiled_obj.parent_context
+        self.string_name = name
+        self._default = default
+
+    def get_kind(self):
+        return Parameter.POSITIONAL_ONLY
+
+    def to_string(self):
+        string = self.string_name
+        if self._default:
+            string += '=' + self._default
+        return string
+
+    def infer(self):
+        return NO_CONTEXTS
+
+
+class CompiledContextName(ContextNameMixin, AbstractNameDefinition):
+    def __init__(self, context, name):
+        self.string_name = name
+        self._context = context
+        self.parent_context = context.parent_context
+
+
+class EmptyCompiledName(AbstractNameDefinition):
+    """
+    Accessing some names will raise an exception. To avoid not having any
+    completions, just give Jedi the option to return this object. It infers to
+    nothing.
+    """
+    def __init__(self, evaluator, name):
+        self.parent_context = evaluator.builtins_module
+        self.string_name = name
+
+    def infer(self):
+        return NO_CONTEXTS
+
+
+class CompiledObjectFilter(AbstractFilter):
+    name_class = CompiledName
+
+    def __init__(self, evaluator, compiled_object, is_instance=False):
+        self._evaluator = evaluator
+        self.compiled_object = compiled_object
+        self.is_instance = is_instance
+
+    def get(self, name):
+        return self._get(
+            name,
+            lambda: self.compiled_object.access_handle.is_allowed_getattr(name),
+            lambda: self.compiled_object.access_handle.dir(),
+            check_has_attribute=True
+        )
+
+    def _get(self, name, allowed_getattr_callback, dir_callback, check_has_attribute=False):
+        """
+        To remove quite a few access calls we introduced the callback here.
+        """
+        has_attribute, is_descriptor = allowed_getattr_callback()
+        if check_has_attribute and not has_attribute:
+            return []
+
+        # Always use unicode objects in Python 2 from here.
+        name = force_unicode(name)
+
+        if (is_descriptor and not self._evaluator.allow_descriptor_getattr) or not has_attribute:
+            return [self._get_cached_name(name, is_empty=True)]
+
+        if self.is_instance and name not in dir_callback():
+            return []
+        return [self._get_cached_name(name)]
+
+    @memoize_method
+    def _get_cached_name(self, name, is_empty=False):
+        if is_empty:
+            return EmptyCompiledName(self._evaluator, name)
+        else:
+            return self._create_name(name)
+
+    def values(self):
+        from jedi.evaluate.compiled import builtin_from_name
+        names = []
+        needs_type_completions, dir_infos = self.compiled_object.access_handle.get_dir_infos()
+        for name in dir_infos:
+            names += self._get(
+                name,
+                lambda: dir_infos[name],
+                lambda: dir_infos.keys(),
+            )
+
+        # ``dir`` doesn't include the type names.
+        if not self.is_instance and needs_type_completions:
+            for filter in builtin_from_name(self._evaluator, u'type').get_filters():
+                names += filter.values()
+        return names
+
+    def _create_name(self, name):
+        return self.name_class(self._evaluator, self.compiled_object, name)
+
+    def __repr__(self):
+        return "<%s: %s>" % (self.__class__.__name__, self.compiled_object)
+
+
+docstr_defaults = {
+    'floating point number': u'float',
+    'character': u'str',
+    'integer': u'int',
+    'dictionary': u'dict',
+    'string': u'str',
+}
+
+
+def _parse_function_doc(doc):
+    """
+    Takes a function and returns the params and return value as a tuple.
+    This is nothing more than a docstring parser.
+
+    TODO docstrings like utime(path, (atime, mtime)) and a(b [, b]) -> None
+    TODO docstrings like 'tuple of integers'
+    """
+    doc = force_unicode(doc)
+    # parse round parentheses: def func(a, (b,c))
+    try:
+        count = 0
+        start = doc.index('(')
+        for i, s in enumerate(doc[start:]):
+            if s == '(':
+                count += 1
+            elif s == ')':
+                count -= 1
+            if count == 0:
+                end = start + i
+                break
+        param_str = doc[start + 1:end]
+    except (ValueError, UnboundLocalError):
+        # ValueError for doc.index
+        # UnboundLocalError for undefined end in last line
+        debug.dbg('no brackets found - no param')
+        end = 0
+        param_str = u''
+    else:
+        # remove square brackets, that show an optional param ( = None)
+        def change_options(m):
+            args = m.group(1).split(',')
+            for i, a in enumerate(args):
+                if a and '=' not in a:
+                    args[i] += '=None'
+            return ','.join(args)
+
+        while True:
+            param_str, changes = re.subn(r' ?\[([^\[\]]+)\]',
+                                         change_options, param_str)
+            if changes == 0:
+                break
+    param_str = param_str.replace('-', '_')  # see: isinstance.__doc__
+
+    # parse return value
+    r = re.search(u'-[>-]* ', doc[end:end + 7])
+    if r is None:
+        ret = u''
+    else:
+        index = end + r.end()
+        # get result type, which can contain newlines
+        pattern = re.compile(r'(,\n|[^\n-])+')
+        ret_str = pattern.match(doc, index).group(0).strip()
+        # New object -> object()
+        ret_str = re.sub(r'[nN]ew (.*)', r'\1()', ret_str)
+
+        ret = docstr_defaults.get(ret_str, ret_str)
+
+    return param_str, ret
+
+
+def _create_from_name(evaluator, compiled_object, name):
+    access_paths = compiled_object.access_handle.getattr_paths(name, default=None)
+    parent_context = compiled_object
+    if parent_context.is_class():
+        parent_context = parent_context.parent_context
+
+    context = None
+    for access_path in access_paths:
+        context = create_cached_compiled_object(
+            evaluator, access_path, parent_context=context
+        )
+    return context
+
+
+def _normalize_create_args(func):
+    """The cache doesn't care about keyword vs. normal args."""
+    def wrapper(evaluator, obj, parent_context=None):
+        return func(evaluator, obj, parent_context)
+    return wrapper
+
+
+def create_from_access_path(evaluator, access_path):
+    parent_context = None
+    for name, access in access_path.accesses:
+        parent_context = create_cached_compiled_object(evaluator, access, parent_context)
+    return parent_context
+
+
+@_normalize_create_args
+@evaluator_function_cache()
+def create_cached_compiled_object(evaluator, access_handle, parent_context):
+    return CompiledObject(evaluator, access_handle, parent_context)

jedi/evaluate/compiled/fake.py

@@ -1,214 +0,0 @@
-"""
-Loads functions that are mixed in to the standard library. E.g. builtins are
-written in C (binaries), but my autocompletion only understands Python code. By
-mixing in Python code, the autocompletion should work much better for builtins.
-"""
-
-import os
-import inspect
-import types
-
-from jedi._compatibility import is_py3, builtins, unicode, is_py34
-from jedi.parser import ParserWithRecovery, load_grammar
-from jedi.parser import tree as pt
-
-modules = {}
-
-
-MethodDescriptorType = type(str.replace)
-# These are not considered classes and access is granted even though they have
-# a __class__ attribute.
-NOT_CLASS_TYPES = (
-    types.BuiltinFunctionType,
-    types.CodeType,
-    types.FrameType,
-    types.FunctionType,
-    types.GeneratorType,
-    types.GetSetDescriptorType,
-    types.LambdaType,
-    types.MemberDescriptorType,
-    types.MethodType,
-    types.ModuleType,
-    types.TracebackType,
-    MethodDescriptorType
-)
-
-if is_py3:
-    NOT_CLASS_TYPES += (
-        types.MappingProxyType,
-        types.SimpleNamespace
-    )
-    if is_py34:
-        NOT_CLASS_TYPES += (types.DynamicClassAttribute,)
-
-
-class FakeDoesNotExist(Exception):
-    pass
-
-
-def _load_faked_module(module):
-    module_name = module.__name__
-    if module_name == '__builtin__' and not is_py3:
-        module_name = 'builtins'
-
-    try:
-        return modules[module_name]
-    except KeyError:
-        path = os.path.dirname(os.path.abspath(__file__))
-        try:
-            with open(os.path.join(path, 'fake', module_name) + '.pym') as f:
-                source = f.read()
-        except IOError:
-            modules[module_name] = None
-            return
-        grammar = load_grammar(version='3.4')
-        module = ParserWithRecovery(grammar, unicode(source), module_name).module
-        modules[module_name] = module
-
-        if module_name == 'builtins' and not is_py3:
-            # There are two implementations of `open` for either python 2/3.
-            # -> Rename the python2 version (`look at fake/builtins.pym`).
-            open_func = _search_scope(module, 'open')
-            open_func.children[1].value = 'open_python3'
-            open_func = _search_scope(module, 'open_python2')
-            open_func.children[1].value = 'open'
-        return module
-
-
-def _search_scope(scope, obj_name):
-    for s in scope.subscopes:
-        if s.name.value == obj_name:
-            return s
-
-
-def get_module(obj):
-    if inspect.ismodule(obj):
-        return obj
-    try:
-        obj = obj.__objclass__
-    except AttributeError:
-        pass
-
-    try:
-        imp_plz = obj.__module__
-    except AttributeError:
-        # Unfortunately in some cases like `int` there's no __module__
-        return builtins
-    else:
-        if imp_plz is None:
-            # Happens for example in `(_ for _ in []).send.__module__`.
-            return builtins
-        else:
-            try:
-                return __import__(imp_plz)
-            except ImportError:
-                # __module__ can be something arbitrary that doesn't exist.
-                return builtins
-
-
-def _faked(module, obj, name):
-    # Crazy underscore actions to try to escape all the internal madness.
-    if module is None:
-        module = get_module(obj)
-
-    faked_mod = _load_faked_module(module)
-    if faked_mod is None:
-        return None, None
-
-    # Having the module as a `parser.tree.Module`, we need to scan
-    # for methods.
-    if name is None:
-        if inspect.isbuiltin(obj) or inspect.isclass(obj):
-            return _search_scope(faked_mod, obj.__name__), faked_mod
-        elif not inspect.isclass(obj):
-            # object is a method or descriptor
-            try:
-                objclass = obj.__objclass__
-            except AttributeError:
-                return None, None
-            else:
-                cls = _search_scope(faked_mod, objclass.__name__)
-                if cls is None:
-                    return None, None
-                return _search_scope(cls, obj.__name__), faked_mod
-    else:
-        if obj == module:
-            return _search_scope(faked_mod, name), faked_mod
-        else:
-            try:
-                cls_name = obj.__name__
-            except AttributeError:
-                return None, None
-            cls = _search_scope(faked_mod, cls_name)
-            if cls is None:
-                return None, None
-            return _search_scope(cls, name), faked_mod
-    return None, None
-
-
-def memoize_faked(obj):
-    """
-    A typical memoize function that ignores issues with non hashable results.
-    """
-    cache = obj.cache = {}
-
-    def memoizer(*args, **kwargs):
-        key = (obj, args, frozenset(kwargs.items()))
-        try:
-            result = cache[key]
-        except TypeError:
-            return obj(*args, **kwargs)
-        except KeyError:
-            result = obj(*args, **kwargs)
-            if result is not None:
-                cache[key] = obj(*args, **kwargs)
-            return result
-        else:
-            return result
-    return memoizer
-
-
-@memoize_faked
-def _get_faked(module, obj, name=None):
-    result, fake_module = _faked(module, obj, name)
-    if result is None:
-        # We're not interested in classes. What we want is functions.
-        raise FakeDoesNotExist
-    elif result.type == 'classdef':
-        return result, fake_module
-    else:
-        # Set the docstr which was previously not set (faked modules don't
-        # contain it).
-        assert result.type == 'funcdef'
-        doc = '"""%s"""' % obj.__doc__  # TODO need escapes.
-        suite = result.children[-1]
-        string = pt.String(doc, (0, 0), '')
-        new_line = pt.Newline('\n', (0, 0))
-        docstr_node = pt.Node('simple_stmt', [string, new_line])
-        suite.children.insert(1, docstr_node)
-        return result, fake_module
-
-
-def get_faked(evaluator, module, obj, name=None, parent_context=None):
-    if parent_context and parent_context.tree_node is not None:
-        # Try to search in already clearly defined stuff.
-        found = _search_scope(parent_context.tree_node, name)
-        if found is not None:
-            return found
-        else:
-            raise FakeDoesNotExist
-
-    faked, fake_module = _get_faked(module and module.obj, obj, name)
-    if module is not None:
-        module.used_names = fake_module.used_names
-    return faked
-
-
-def is_class_instance(obj):
-    """Like inspect.* methods."""
-    try:
-        cls = obj.__class__
-    except AttributeError:
-        return False
-    else:
-        return cls != type and not issubclass(cls, NOT_CLASS_TYPES)

jedi/evaluate/compiled/fake/_functools.pym

@@ -1,9 +0,0 @@
-class partial():
-    def __init__(self, func, *args, **keywords):
-        self.__func = func
-        self.__args = args
-        self.__keywords = keywords
-
-    def __call__(self, *args, **kwargs):
-        # TODO should be **dict(self.__keywords, **kwargs)
-        return self.__func(*(self.__args + args), **self.__keywords)

jedi/evaluate/compiled/fake/_sqlite3.pym

@@ -1,26 +0,0 @@
-def connect(database, timeout=None, isolation_level=None, detect_types=None, factory=None):
-    return Connection()
-
-
-class Connection():
-    def cursor(self):
-        return Cursor()
-
-
-class Cursor():
-    def cursor(self):
-        return Cursor()
-
-    def fetchone(self):
-        return Row()
-
-    def fetchmany(self, size=cursor.arraysize):
-        return [self.fetchone()]
-
-    def fetchall(self):
-        return [self.fetchone()]
-
-
-class Row():
-    def keys(self):
-        return ['']

jedi/evaluate/compiled/fake/_sre.pym

@@ -1,99 +0,0 @@
-def compile():
-    class SRE_Match():
-        endpos = int()
-        lastgroup = int()
-        lastindex = int()
-        pos = int()
-        string = str()
-        regs = ((int(), int()),)
-
-        def __init__(self, pattern):
-            self.re = pattern
-
-        def start(self):
-            return int()
-
-        def end(self):
-            return int()
-
-        def span(self):
-            return int(), int()
-
-        def expand(self):
-            return str()
-
-        def group(self, nr):
-            return str()
-
-        def groupdict(self):
-            return {str(): str()}
-
-        def groups(self):
-            return (str(),)
-
-    class SRE_Pattern():
-        flags = int()
-        groupindex = {}
-        groups = int()
-        pattern = str()
-
-        def findall(self, string, pos=None, endpos=None):
-            """
-            findall(string[, pos[, endpos]]) --> list.
-            Return a list of all non-overlapping matches of pattern in string.
-            """
-            return [str()]
-
-        def finditer(self, string, pos=None, endpos=None):
-            """
-            finditer(string[, pos[, endpos]]) --> iterator.
-            Return an iterator over all non-overlapping matches for the
-            RE pattern in string. For each match, the iterator returns a
-            match object.
-            """
-            yield SRE_Match(self)
-
-        def match(self, string, pos=None, endpos=None):
-            """
-            match(string[, pos[, endpos]]) --> match object or None.
-            Matches zero or more characters at the beginning of the string
-            pattern
-            """
-            return SRE_Match(self)
-
-        def scanner(self, string, pos=None, endpos=None):
-            pass
-
-        def search(self, string, pos=None, endpos=None):
-            """
-            search(string[, pos[, endpos]]) --> match object or None.
-            Scan through string looking for a match, and return a corresponding
-            MatchObject instance. Return None if no position in the string matches.
-            """
-            return SRE_Match(self)
-
-        def split(self, string, maxsplit=0]):
-            """
-            split(string[, maxsplit = 0])  --> list.
-            Split string by the occurrences of pattern.
-            """
-            return [str()]
-
-        def sub(self, repl, string, count=0):
-            """
-            sub(repl, string[, count = 0]) --> newstring
-            Return the string obtained by replacing the leftmost non-overlapping
-            occurrences of pattern in string by the replacement repl.
-            """
-            return str()
-
-        def subn(self, repl, string, count=0):
-            """
-            subn(repl, string[, count = 0]) --> (newstring, number of subs)
-            Return the tuple (new_string, number_of_subs_made) found by replacing
-            the leftmost non-overlapping occurrences of pattern with the
-            replacement repl.
-            """
-            return (str(), int())
-
-    return SRE_Pattern()

jedi/evaluate/compiled/fake/_weakref.pym

@@ -1,9 +0,0 @@
-def proxy(object, callback=None):
-    return object
-
-class ref():
-    def __init__(self, object, callback=None):
-        self.__object = object
-
-    def __call__(self):
-        return self.__object

jedi/evaluate/compiled/fake/builtins.pym

@@ -1,267 +0,0 @@
-"""
-Pure Python implementation of some builtins.
-This code is not going to be executed anywhere.
-These implementations are not always correct, but should work as good as
-possible for the auto completion.
-"""
-
-
-def next(iterator, default=None):
-    if random.choice([0, 1]):
-        if hasattr("next"):
-            return iterator.next()
-        else:
-            return iterator.__next__()
-    else:
-        if default is not None:
-            return default
-
-
-def iter(collection, sentinel=None):
-    if sentinel:
-        yield collection()
-    else:
-        for c in collection:
-            yield c
-
-
-def range(start, stop=None, step=1):
-    return [0]
-
-
-class file():
-    def __iter__(self):
-        yield ''
-    def next(self):
-        return ''
-
-
-class xrange():
-    # Attention: this function doesn't exist in Py3k (there it is range).
-    def __iter__(self):
-        yield 1
-
-    def count(self):
-        return 1
-
-    def index(self):
-        return 1
-
-
-def open(file, mode='r', buffering=-1, encoding=None, errors=None, newline=None, closefd=True):
-    import io
-    return io.TextIOWrapper(file, mode, buffering, encoding, errors, newline, closefd)
-
-
-def open_python2(name, mode=None, buffering=None):
-    return file(name, mode, buffering)
-
-
-#--------------------------------------------------------
-# descriptors
-#--------------------------------------------------------
-class property():
-    def __init__(self, fget, fset=None, fdel=None, doc=None):
-        self.fget = fget
-        self.fset = fset
-        self.fdel = fdel
-        self.__doc__ = doc
-
-    def __get__(self, obj, cls):
-        return self.fget(obj)
-
-    def __set__(self, obj, value):
-        self.fset(obj, value)
-
-    def __delete__(self, obj):
-        self.fdel(obj)
-
-    def setter(self, func):
-        self.fset = func
-        return self
-
-    def getter(self, func):
-        self.fget = func
-        return self
-
-    def deleter(self, func):
-        self.fdel = func
-        return self
-
-
-class staticmethod():
-    def __init__(self, func):
-        self.__func = func
-
-    def __get__(self, obj, cls):
-        return self.__func
-
-
-class classmethod():
-    def __init__(self, func):
-        self.__func = func
-
-    def __get__(self, obj, cls):
-        def _method(*args, **kwargs):
-            return self.__func(cls, *args, **kwargs)
-        return _method
-
-
-#--------------------------------------------------------
-# array stuff
-#--------------------------------------------------------
-class list():
-    def __init__(self, iterable=[]):
-        self.__iterable = []
-        for i in iterable:
-            self.__iterable += [i]
-
-    def __iter__(self):
-        for i in self.__iterable:
-            yield i
-
-    def __getitem__(self, y):
-        return self.__iterable[y]
-
-    def pop(self):
-        return self.__iterable[int()]
-
-
-class tuple():
-    def __init__(self, iterable=[]):
-        self.__iterable = []
-        for i in iterable:
-            self.__iterable += [i]
-
-    def __iter__(self):
-        for i in self.__iterable:
-            yield i
-
-    def __getitem__(self, y):
-        return self.__iterable[y]
-
-    def index(self):
-        return 1
-
-    def count(self):
-        return 1
-
-
-class set():
-    def __init__(self, iterable=[]):
-        self.__iterable = iterable
-
-    def __iter__(self):
-        for i in self.__iterable:
-            yield i
-
-    def pop(self):
-        return list(self.__iterable)[-1]
-
-    def copy(self):
-        return self
-
-    def difference(self, other):
-        return self - other
-
-    def intersection(self, other):
-        return self & other
-
-    def symmetric_difference(self, other):
-        return self ^ other
-
-    def union(self, other):
-        return self | other
-
-
-class frozenset():
-    def __init__(self, iterable=[]):
-        self.__iterable = iterable
-
-    def __iter__(self):
-        for i in self.__iterable:
-            yield i
-
-    def copy(self):
-        return self
-
-
-class dict():
-    def __init__(self, **elements):
-        self.__elements = elements
-
-    def clear(self):
-        # has a strange docstr
-        pass
-
-    def get(self, k, d=None):
-        # TODO implement
-        try:
-            #return self.__elements[k]
-            pass
-        except KeyError:
-            return d
-
-    def values(self):
-        return self.__elements.values()
-
-    def setdefault(self, k, d):
-        # TODO maybe also return the content
-        return d
-
-
-class enumerate():
-    def __init__(self, sequence, start=0):
-        self.__sequence = sequence
-
-    def __iter__(self):
-        for i in self.__sequence:
-            yield 1, i
-
-    def __next__(self):
-        return next(self.__iter__())
-
-    def next(self):
-        return next(self.__iter__())
-
-
-class reversed():
-    def __init__(self, sequence):
-        self.__sequence = sequence
-
-    def __iter__(self):
-        for i in self.__sequence:
-            yield i
-
-    def __next__(self):
-        return next(self.__iter__())
-
-    def next(self):
-        return next(self.__iter__())
-
-
-def sorted(iterable, cmp=None, key=None, reverse=False):
-    return iterable
-
-
-#--------------------------------------------------------
-# basic types
-#--------------------------------------------------------
-class int():
-    def __init__(self, x, base=None):
-        pass
-
-
-class str():
-    def __init__(self, obj):
-        pass
-
-    def strip(self):
-        return str()
-
-    def split(self):
-        return [str()]
-
-class type():
-    def mro():
-        return [object]

jedi/evaluate/compiled/fake/datetime.pym

@@ -1,4 +0,0 @@
-class datetime():
-    @staticmethod
-    def now():
-        return datetime()

jedi/evaluate/compiled/fake/io.pym

@@ -1,6 +0,0 @@
-class TextIOWrapper():
-    def __next__(self):
-        return str()
-
-    def __iter__(self):
-        yield str()

jedi/evaluate/compiled/fake/posix.pym

@@ -1,5 +0,0 @@
-def getcwd():
-    return ''
-
-def getcwdu():
-    return ''

jedi/evaluate/compiled/getattr_static.py

@@ -0,0 +1,176 @@
+"""
+A static version of getattr.
+This is a backport of the Python 3 code with a little bit of additional
+information returned to enable Jedi to make decisions.
+"""
+
+import types
+
+from jedi._compatibility import py_version
+
+_sentinel = object()
+
+
+def _check_instance(obj, attr):
+    instance_dict = {}
+    try:
+        instance_dict = object.__getattribute__(obj, "__dict__")
+    except AttributeError:
+        pass
+    return dict.get(instance_dict, attr, _sentinel)
+
+
+def _check_class(klass, attr):
+    for entry in _static_getmro(klass):
+        if _shadowed_dict(type(entry)) is _sentinel:
+            try:
+                return entry.__dict__[attr]
+            except KeyError:
+                pass
+    return _sentinel
+
+
+def _is_type(obj):
+    try:
+        _static_getmro(obj)
+    except TypeError:
+        return False
+    return True
+
+
+def _shadowed_dict_newstyle(klass):
+    dict_attr = type.__dict__["__dict__"]
+    for entry in _static_getmro(klass):
+        try:
+            class_dict = dict_attr.__get__(entry)["__dict__"]
+        except KeyError:
+            pass
+        else:
+            if not (type(class_dict) is types.GetSetDescriptorType and
+                    class_dict.__name__ == "__dict__" and
+                    class_dict.__objclass__ is entry):
+                return class_dict
+    return _sentinel
+
+
+def _static_getmro_newstyle(klass):
+    return type.__dict__['__mro__'].__get__(klass)
+
+
+if py_version >= 30:
+    _shadowed_dict = _shadowed_dict_newstyle
+    _get_type = type
+    _static_getmro = _static_getmro_newstyle
+else:
+    def _shadowed_dict(klass):
+        """
+        In Python 2 __dict__ is not overwritable:
+
+            class Foo(object): pass
+            setattr(Foo, '__dict__', 4)
+
+            Traceback (most recent call last):
+              File "<stdin>", line 1, in <module>
+            TypeError: __dict__ must be a dictionary object
+
+        It applies to both newstyle and oldstyle classes:
+
+            class Foo(object): pass
+            setattr(Foo, '__dict__', 4)
+            Traceback (most recent call last):
+              File "<stdin>", line 1, in <module>
+            AttributeError: attribute '__dict__' of 'type' objects is not writable
+
+        It also applies to instances of those objects. However to keep things
+        straight forward, newstyle classes always use the complicated way of
+        accessing it while oldstyle classes just use getattr.
+        """
+        if type(klass) is _oldstyle_class_type:
+            return getattr(klass, '__dict__', _sentinel)
+        return _shadowed_dict_newstyle(klass)
+
+    class _OldStyleClass:
+        pass
+
+    _oldstyle_instance_type = type(_OldStyleClass())
+    _oldstyle_class_type = type(_OldStyleClass)
+
+    def _get_type(obj):
+        type_ = object.__getattribute__(obj, '__class__')
+        if type_ is _oldstyle_instance_type:
+            # Somehow for old style classes we need to access it directly.
+            return obj.__class__
+        return type_
+
+    def _static_getmro(klass):
+        if type(klass) is _oldstyle_class_type:
+            def oldstyle_mro(klass):
+                """
+                Oldstyle mro is a really simplistic way of look up mro:
+                https://stackoverflow.com/questions/54867/what-is-the-difference-between-old-style-and-new-style-classes-in-python
+                """
+                yield klass
+                for base in klass.__bases__:
+                    for yield_from in oldstyle_mro(base):
+                        yield yield_from
+
+            return oldstyle_mro(klass)
+
+        return _static_getmro_newstyle(klass)
+
+
+def _safe_hasattr(obj, name):
+    return _check_class(_get_type(obj), name) is not _sentinel
+
+
+def _safe_is_data_descriptor(obj):
+    return _safe_hasattr(obj, '__set__') or _safe_hasattr(obj, '__delete__')
+
+
+def getattr_static(obj, attr, default=_sentinel):
+    """Retrieve attributes without triggering dynamic lookup via the
+       descriptor protocol,  __getattr__ or __getattribute__.
+
+       Note: this function may not be able to retrieve all attributes
+       that getattr can fetch (like dynamically created attributes)
+       and may find attributes that getattr can't (like descriptors
+       that raise AttributeError). It can also return descriptor objects
+       instead of instance members in some cases. See the
+       documentation for details.
+
+       Returns a tuple `(attr, is_get_descriptor)`. is_get_descripter means that
+       the attribute is a descriptor that has a `__get__` attribute.
+    """
+    instance_result = _sentinel
+    if not _is_type(obj):
+        klass = _get_type(obj)
+        dict_attr = _shadowed_dict(klass)
+        if (dict_attr is _sentinel or type(dict_attr) is types.MemberDescriptorType):
+            instance_result = _check_instance(obj, attr)
+    else:
+        klass = obj
+
+    klass_result = _check_class(klass, attr)
+
+    if instance_result is not _sentinel and klass_result is not _sentinel:
+        if _safe_hasattr(klass_result, '__get__') \
+                and _safe_is_data_descriptor(klass_result):
+            # A get/set descriptor has priority over everything.
+            return klass_result, True
+
+    if instance_result is not _sentinel:
+        return instance_result, False
+    if klass_result is not _sentinel:
+        return klass_result, _safe_hasattr(klass_result, '__get__')
+
+    if obj is klass:
+        # for types we check the metaclass too
+        for entry in _static_getmro(type(klass)):
+            if _shadowed_dict(type(entry)) is _sentinel:
+                try:
+                    return entry.__dict__[attr], False
+                except KeyError:
+                    pass
+    if default is not _sentinel:
+        return default, False
+    raise AttributeError(attr)

jedi/evaluate/compiled/mixed.py

@@ -4,20 +4,32 @@ Used only for REPL Completion.
 
 import inspect
 import os
+import sys
 
-from jedi import common
-from jedi.parser.diff import FastParser
+from jedi.parser_utils import get_cached_code_lines
+
+from jedi import settings
 from jedi.evaluate import compiled
 from jedi.cache import underscore_memoization
-from jedi.evaluate import imports
-from jedi.evaluate.context import Context
+from jedi.file_io import FileIO
+from jedi.evaluate.base_context import ContextSet, ContextWrapper
+from jedi.evaluate.helpers import SimpleGetItemNotFound
+from jedi.evaluate.context import ModuleContext
+from jedi.evaluate.cache import evaluator_function_cache
+from jedi.evaluate.compiled.getattr_static import getattr_static
+from jedi.evaluate.compiled.access import compiled_objects_cache, \
+    ALLOWED_GETITEM_TYPES, get_api_type
+from jedi.evaluate.compiled.context import create_cached_compiled_object
+from jedi.evaluate.gradual.conversion import to_stub
+
+_sentinel = object()
 
 
-class MixedObject(object):
+class MixedObject(ContextWrapper):
     """
     A ``MixedObject`` is used in two ways:
 
-    1. It uses the default logic of ``parser.tree`` objects,
+    1. It uses the default logic of ``parser.python.tree`` objects,
     2. except for getattr calls. The names dicts are generated in a fashion
        like ``CompiledObject``.
 
@@ -30,43 +42,39 @@ class MixedObject(object):
     fewer special cases, because we in Python you don't have the same freedoms
     to modify the runtime.
     """
-    def __init__(self, evaluator, parent_context, compiled_object, tree_name):
-        self.evaluator = evaluator
+    def __init__(self, compiled_object, tree_context):
+        super(MixedObject, self).__init__(tree_context)
         self.compiled_object = compiled_object
-        self.obj = compiled_object.obj
-        self._tree_name = tree_name
-        name_module = tree_name.get_root_node()
-        if parent_context.tree_node.get_root_node() != name_module:
-            from jedi.evaluate.representation import ModuleContext
-            module_context = ModuleContext(evaluator, name_module)
-            name = compiled_object.get_root_context().py__name__()
-            imports.add_module(evaluator, name, module_context)
-        else:
-            module_context = parent_context.get_root_context()
-
-        self._context = module_context.create_context(
-            tree_name.parent,
-            node_is_context=True,
-            node_is_object=True
-        )
-
-    # We have to overwrite everything that has to do with trailers, name
-    # lookups and filters to make it possible to route name lookups towards
-    # compiled objects and the rest towards tree node contexts.
-    def eval_trailer(*args, **kwags):
-        return Context.eval_trailer(*args, **kwags)
-
-    def py__getattribute__(*args, **kwargs):
-        return Context.py__getattribute__(*args, **kwargs)
+        self.access_handle = compiled_object.access_handle
 
     def get_filters(self, *args, **kwargs):
         yield MixedObjectFilter(self.evaluator, self)
 
-    def __repr__(self):
-        return '<%s: %s>' % (type(self).__name__, repr(self.obj))
+    def get_signatures(self):
+        # Prefer `inspect.signature` over somehow analyzing Python code. It
+        # should be very precise, especially for stuff like `partial`.
+        return self.compiled_object.get_signatures()
 
-    def __getattr__(self, name):
-        return getattr(self._context, name)
+    def py__call__(self, arguments):
+        return (to_stub(self._wrapped_context) or self._wrapped_context).py__call__(arguments)
+
+    def get_safe_value(self, default=_sentinel):
+        if default is _sentinel:
+            return self.compiled_object.get_safe_value()
+        else:
+            return self.compiled_object.get_safe_value(default)
+
+    def py__simple_getitem__(self, index):
+        python_object = self.compiled_object.access_handle.access._obj
+        if type(python_object) in ALLOWED_GETITEM_TYPES:
+            return self.compiled_object.py__simple_getitem__(index)
+        raise SimpleGetItemNotFound
+
+    def __repr__(self):
+        return '<%s: %s>' % (
+            type(self).__name__,
+            self.access_handle.get_repr()
+        )
 
 
 class MixedName(compiled.CompiledName):
@@ -78,7 +86,7 @@ class MixedName(compiled.CompiledName):
         contexts = list(self.infer())
         if not contexts:
             # This means a start_pos that doesn't exist (compiled objects).
-            return (0, 0)
+            return 0, 0
         return contexts[0].name.start_pos
 
     @start_pos.setter
@@ -88,15 +96,21 @@ class MixedName(compiled.CompiledName):
 
     @underscore_memoization
     def infer(self):
-        obj = self.parent_context.obj
-        try:
-            obj = getattr(obj, self.string_name)
-        except AttributeError:
-            # Happens e.g. in properties of
-            # PyQt4.QtGui.QStyleOptionComboBox.currentText
-            # -> just set it to None
-            obj = None
-        return [create(self._evaluator, obj, parent_context=self.parent_context)]
+        # TODO use logic from compiled.CompiledObjectFilter
+        access_paths = self.parent_context.access_handle.getattr_paths(
+            self.string_name,
+            default=None
+        )
+        assert len(access_paths)
+        contexts = [None]
+        for access in access_paths:
+            contexts = ContextSet.from_sets(
+                _create(self._evaluator, access, parent_context=c)
+                if c is None or isinstance(c, MixedObject)
+                else ContextSet({create_cached_compiled_object(c.evaluator, access, c)})
+                for c in contexts
+            )
+        return contexts
 
     @property
     def api_type(self):
@@ -106,32 +120,45 @@ class MixedName(compiled.CompiledName):
 class MixedObjectFilter(compiled.CompiledObjectFilter):
     name_class = MixedName
 
-    def __init__(self, evaluator, mixed_object, is_instance=False):
-        super(MixedObjectFilter, self).__init__(
-            evaluator, mixed_object, is_instance)
-        self._mixed_object = mixed_object
 
-    #def _create(self, name):
-        #return MixedName(self._evaluator, self._compiled_object, name)
+@evaluator_function_cache()
+def _load_module(evaluator, path):
+    module_node = evaluator.parse(
+        path=path,
+        cache=True,
+        diff_cache=settings.fast_parser,
+        cache_path=settings.cache_directory
+    ).get_root_node()
+    # python_module = inspect.getmodule(python_object)
+    # TODO we should actually make something like this possible.
+    #evaluator.modules[python_module.__name__] = module_node
+    return module_node
 
 
-def parse(grammar, path):
-    with open(path) as f:
-        source = f.read()
-    source = common.source_to_unicode(source)
-    return FastParser(grammar, source, path)
+def _get_object_to_check(python_object):
+    """Check if inspect.getfile has a chance to find the source."""
+    if sys.version_info[0] > 2:
+        python_object = inspect.unwrap(python_object)
 
+    if (inspect.ismodule(python_object) or
+            inspect.isclass(python_object) or
+            inspect.ismethod(python_object) or
+            inspect.isfunction(python_object) or
+            inspect.istraceback(python_object) or
+            inspect.isframe(python_object) or
+            inspect.iscode(python_object)):
+        return python_object
 
-def _load_module(evaluator, path, python_object):
-    module = parse(evaluator.grammar, path).module
-    python_module = inspect.getmodule(python_object)
-
-    evaluator.modules[python_module.__name__] = module
-    return module
-
-
-def find_syntax_node_name(evaluator, python_object):
     try:
+        return python_object.__class__
+    except AttributeError:
+        raise TypeError  # Prevents computation of `repr` within inspect.
+
+
+def _find_syntax_node_name(evaluator, python_object):
+    original_object = python_object
+    try:
+        python_object = _get_object_to_check(python_object)
         path = inspect.getsourcefile(python_object)
     except TypeError:
         # The type might not be known (e.g. class_with_dict.__weakref__)
@@ -140,24 +167,36 @@ def find_syntax_node_name(evaluator, python_object):
         # The path might not exist or be e.g. <stdin>.
         return None
 
-    module = _load_module(evaluator, path, python_object)
+    file_io = FileIO(path)
+    module_node = _load_module(evaluator, path)
 
     if inspect.ismodule(python_object):
         # We don't need to check names for modules, because there's not really
         # a way to write a module in a module in Python (and also __name__ can
         # be something like ``email.utils``).
-        return module.name
+        code_lines = get_cached_code_lines(evaluator.grammar, path)
+        return module_node, module_node, file_io, code_lines
+
+    try:
+        name_str = python_object.__name__
+    except AttributeError:
+        # Stuff like python_function.__code__.
+        return None
 
-    name_str = python_object.__name__
     if name_str == '<lambda>':
         return None  # It's too hard to find lambdas.
 
     # Doesn't always work (e.g. os.stat_result)
-    try:
-        names = module.used_names[name_str]
-    except KeyError:
+    names = module_node.get_used_names().get(name_str, [])
+    # Only functions and classes are relevant. If a name e.g. points to an
+    # import, it's probably a builtin (like collections.deque) and needs to be
+    # ignored.
+    names = [
+        n for n in names
+        if n.parent.type in ('funcdef', 'classdef') and n.parent.name == n
+    ]
+    if not names:
         return None
-    names = [n for n in names if n.is_definition()]
 
     try:
         code = python_object.__code__
@@ -173,22 +212,80 @@ def find_syntax_node_name(evaluator, python_object):
         # There's a chance that the object is not available anymore, because
         # the code has changed in the background.
         if line_names:
-            return line_names[-1]
+            names = line_names
 
+    code_lines = get_cached_code_lines(evaluator.grammar, path)
     # It's really hard to actually get the right definition, here as a last
     # resort we just return the last one. This chance might lead to odd
     # completions at some points but will lead to mostly correct type
     # inference, because people tend to define a public name in a module only
     # once.
-    return names[-1]
+    tree_node = names[-1].parent
+    if tree_node.type == 'funcdef' and get_api_type(original_object) == 'instance':
+        # If an instance is given and we're landing on a function (e.g.
+        # partial in 3.5), something is completely wrong and we should not
+        # return that.
+        return None
+    return module_node, tree_node, file_io, code_lines
 
 
-@compiled.compiled_objects_cache('mixed_cache')
-def create(evaluator, obj, parent_context=None, *args):
-    tree_name = find_syntax_node_name(evaluator, obj)
+@compiled_objects_cache('mixed_cache')
+def _create(evaluator, access_handle, parent_context, *args):
+    compiled_object = create_cached_compiled_object(
+        evaluator,
+        access_handle,
+        parent_context=parent_context and parent_context.compiled_object
+    )
 
-    compiled_object = compiled.create(
-        evaluator, obj, parent_context=parent_context.compiled_object)
-    if tree_name is None:
-        return compiled_object
-    return MixedObject(evaluator, parent_context, compiled_object, tree_name)
+    # TODO accessing this is bad, but it probably doesn't matter that much,
+    # because we're working with interpreteters only here.
+    python_object = access_handle.access._obj
+    result = _find_syntax_node_name(evaluator, python_object)
+    if result is None:
+        # TODO Care about generics from stuff like `[1]` and don't return like this.
+        if type(python_object) in (dict, list, tuple):
+            return ContextSet({compiled_object})
+
+        tree_contexts = to_stub(compiled_object)
+        if not tree_contexts:
+            return ContextSet({compiled_object})
+    else:
+        module_node, tree_node, file_io, code_lines = result
+
+        if parent_context is None:
+            # TODO this __name__ is probably wrong.
+            name = compiled_object.get_root_context().py__name__()
+            string_names = tuple(name.split('.'))
+            module_context = ModuleContext(
+                evaluator, module_node,
+                file_io=file_io,
+                string_names=string_names,
+                code_lines=code_lines,
+                is_package=hasattr(compiled_object, 'py__path__'),
+            )
+            if name is not None:
+                evaluator.module_cache.add(string_names, ContextSet([module_context]))
+        else:
+            if parent_context.tree_node.get_root_node() != module_node:
+                # This happens e.g. when __module__ is wrong, or when using
+                # TypeVar('foo'), where Jedi uses 'foo' as the name and
+                # Python's TypeVar('foo').__module__ will be typing.
+                return ContextSet({compiled_object})
+            module_context = parent_context.get_root_context()
+
+        tree_contexts = ContextSet({
+            module_context.create_context(
+                tree_node,
+                node_is_context=True,
+                node_is_object=True
+            )
+        })
+        if tree_node.type == 'classdef':
+            if not access_handle.is_class():
+                # Is an instance, not a class.
+                tree_contexts = tree_contexts.execute_evaluated()
+
+    return ContextSet(
+        MixedObject(compiled_object, tree_context=tree_context)
+        for tree_context in tree_contexts
+    )

jedi/evaluate/compiled/subprocess/__init__.py

@@ -0,0 +1,406 @@
+"""
+Makes it possible to do the compiled analysis in a subprocess. This has two
+goals:
+
+1. Making it safer - Segfaults and RuntimeErrors as well as stdout/stderr can
+   be ignored and dealt with.
+2. Make it possible to handle different Python versions as well as virtualenvs.
+"""
+
+import os
+import sys
+import subprocess
+import socket
+import errno
+import traceback
+from functools import partial
+from threading import Thread
+try:
+    from queue import Queue, Empty
+except ImportError:
+    from Queue import Queue, Empty  # python 2.7
+
+from jedi._compatibility import queue, is_py3, force_unicode, \
+    pickle_dump, pickle_load, GeneralizedPopen, weakref
+from jedi import debug
+from jedi.cache import memoize_method
+from jedi.evaluate.compiled.subprocess import functions
+from jedi.evaluate.compiled.access import DirectObjectAccess, AccessPath, \
+    SignatureParam
+from jedi.api.exceptions import InternalError
+
+
+_MAIN_PATH = os.path.join(os.path.dirname(__file__), '__main__.py')
+
+
+def _enqueue_output(out, queue):
+    for line in iter(out.readline, b''):
+        queue.put(line)
+
+
+def _add_stderr_to_debug(stderr_queue):
+    while True:
+        # Try to do some error reporting from the subprocess and print its
+        # stderr contents.
+        try:
+            line = stderr_queue.get_nowait()
+            line = line.decode('utf-8', 'replace')
+            debug.warning('stderr output: %s' % line.rstrip('\n'))
+        except Empty:
+            break
+
+
+def _get_function(name):
+    return getattr(functions, name)
+
+
+def _cleanup_process(process, thread):
+    try:
+        process.kill()
+        process.wait()
+    except OSError:
+        # Raised if the process is already killed.
+        pass
+    thread.join()
+    for stream in [process.stdin, process.stdout, process.stderr]:
+        try:
+            stream.close()
+        except OSError:
+            # Raised if the stream is broken.
+            pass
+
+
+class _EvaluatorProcess(object):
+    def __init__(self, evaluator):
+        self._evaluator_weakref = weakref.ref(evaluator)
+        self._evaluator_id = id(evaluator)
+        self._handles = {}
+
+    def get_or_create_access_handle(self, obj):
+        id_ = id(obj)
+        try:
+            return self.get_access_handle(id_)
+        except KeyError:
+            access = DirectObjectAccess(self._evaluator_weakref(), obj)
+            handle = AccessHandle(self, access, id_)
+            self.set_access_handle(handle)
+            return handle
+
+    def get_access_handle(self, id_):
+        return self._handles[id_]
+
+    def set_access_handle(self, handle):
+        self._handles[handle.id] = handle
+
+
+class EvaluatorSameProcess(_EvaluatorProcess):
+    """
+    Basically just an easy access to functions.py. It has the same API
+    as EvaluatorSubprocess and does the same thing without using a subprocess.
+    This is necessary for the Interpreter process.
+    """
+    def __getattr__(self, name):
+        return partial(_get_function(name), self._evaluator_weakref())
+
+
+class EvaluatorSubprocess(_EvaluatorProcess):
+    def __init__(self, evaluator, compiled_subprocess):
+        super(EvaluatorSubprocess, self).__init__(evaluator)
+        self._used = False
+        self._compiled_subprocess = compiled_subprocess
+
+    def __getattr__(self, name):
+        func = _get_function(name)
+
+        def wrapper(*args, **kwargs):
+            self._used = True
+
+            result = self._compiled_subprocess.run(
+                self._evaluator_weakref(),
+                func,
+                args=args,
+                kwargs=kwargs,
+            )
+            # IMO it should be possible to create a hook in pickle.load to
+            # mess with the loaded objects. However it's extremely complicated
+            # to work around this so just do it with this call. ~ dave
+            return self._convert_access_handles(result)
+
+        return wrapper
+
+    def _convert_access_handles(self, obj):
+        if isinstance(obj, SignatureParam):
+            return SignatureParam(*self._convert_access_handles(tuple(obj)))
+        elif isinstance(obj, tuple):
+            return tuple(self._convert_access_handles(o) for o in obj)
+        elif isinstance(obj, list):
+            return [self._convert_access_handles(o) for o in obj]
+        elif isinstance(obj, AccessHandle):
+            try:
+                # Rewrite the access handle to one we're already having.
+                obj = self.get_access_handle(obj.id)
+            except KeyError:
+                obj.add_subprocess(self)
+                self.set_access_handle(obj)
+        elif isinstance(obj, AccessPath):
+            return AccessPath(self._convert_access_handles(obj.accesses))
+        return obj
+
+    def __del__(self):
+        if self._used and not self._compiled_subprocess.is_crashed:
+            self._compiled_subprocess.delete_evaluator(self._evaluator_id)
+
+
+class CompiledSubprocess(object):
+    is_crashed = False
+    # Start with 2, gets set after _get_info.
+    _pickle_protocol = 2
+
+    def __init__(self, executable):
+        self._executable = executable
+        self._evaluator_deletion_queue = queue.deque()
+        self._cleanup_callable = lambda: None
+
+    def __repr__(self):
+        pid = os.getpid()
+        return '<%s _executable=%r, _pickle_protocol=%r, is_crashed=%r, pid=%r>' % (
+            self.__class__.__name__,
+            self._executable,
+            self._pickle_protocol,
+            self.is_crashed,
+            pid,
+        )
+
+    @memoize_method
+    def _get_process(self):
+        debug.dbg('Start environment subprocess %s', self._executable)
+        parso_path = sys.modules['parso'].__file__
+        args = (
+            self._executable,
+            _MAIN_PATH,
+            os.path.dirname(os.path.dirname(parso_path)),
+            '.'.join(str(x) for x in sys.version_info[:3]),
+        )
+        process = GeneralizedPopen(
+            args,
+            stdin=subprocess.PIPE,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            # Use system default buffering on Python 2 to improve performance
+            # (this is already the case on Python 3).
+            bufsize=-1
+        )
+        self._stderr_queue = Queue()
+        self._stderr_thread = t = Thread(
+            target=_enqueue_output,
+            args=(process.stderr, self._stderr_queue)
+        )
+        t.daemon = True
+        t.start()
+        # Ensure the subprocess is properly cleaned up when the object
+        # is garbage collected.
+        self._cleanup_callable = weakref.finalize(self,
+                                                  _cleanup_process,
+                                                  process,
+                                                  t)
+        return process
+
+    def run(self, evaluator, function, args=(), kwargs={}):
+        # Delete old evaluators.
+        while True:
+            try:
+                evaluator_id = self._evaluator_deletion_queue.pop()
+            except IndexError:
+                break
+            else:
+                self._send(evaluator_id, None)
+
+        assert callable(function)
+        return self._send(id(evaluator), function, args, kwargs)
+
+    def get_sys_path(self):
+        return self._send(None, functions.get_sys_path, (), {})
+
+    def _kill(self):
+        self.is_crashed = True
+        self._cleanup_callable()
+
+    def _send(self, evaluator_id, function, args=(), kwargs={}):
+        if self.is_crashed:
+            raise InternalError("The subprocess %s has crashed." % self._executable)
+
+        if not is_py3:
+            # Python 2 compatibility
+            kwargs = {force_unicode(key): value for key, value in kwargs.items()}
+
+        data = evaluator_id, function, args, kwargs
+        try:
+            pickle_dump(data, self._get_process().stdin, self._pickle_protocol)
+        except (socket.error, IOError) as e:
+            # Once Python2 will be removed we can just use `BrokenPipeError`.
+            # Also, somehow in windows it returns EINVAL instead of EPIPE if
+            # the subprocess dies.
+            if e.errno not in (errno.EPIPE, errno.EINVAL):
+                # Not a broken pipe
+                raise
+            self._kill()
+            raise InternalError("The subprocess %s was killed. Maybe out of memory?"
+                                % self._executable)
+
+        try:
+            is_exception, traceback, result = pickle_load(self._get_process().stdout)
+        except EOFError as eof_error:
+            try:
+                stderr = self._get_process().stderr.read().decode('utf-8', 'replace')
+            except Exception as exc:
+                stderr = '<empty/not available (%r)>' % exc
+            self._kill()
+            _add_stderr_to_debug(self._stderr_queue)
+            raise InternalError(
+                "The subprocess %s has crashed (%r, stderr=%s)." % (
+                    self._executable,
+                    eof_error,
+                    stderr,
+                ))
+
+        _add_stderr_to_debug(self._stderr_queue)
+
+        if is_exception:
+            # Replace the attribute error message with a the traceback. It's
+            # way more informative.
+            result.args = (traceback,)
+            raise result
+        return result
+
+    def delete_evaluator(self, evaluator_id):
+        """
+        Currently we are not deleting evalutors instantly. They only get
+        deleted once the subprocess is used again. It would probably a better
+        solution to move all of this into a thread. However, the memory usage
+        of a single evaluator shouldn't be that high.
+        """
+        # With an argument - the evaluator gets deleted.
+        self._evaluator_deletion_queue.append(evaluator_id)
+
+
+class Listener(object):
+    def __init__(self, pickle_protocol):
+        self._evaluators = {}
+        # TODO refactor so we don't need to process anymore just handle
+        # controlling.
+        self._process = _EvaluatorProcess(Listener)
+        self._pickle_protocol = pickle_protocol
+
+    def _get_evaluator(self, function, evaluator_id):
+        from jedi.evaluate import Evaluator
+
+        try:
+            evaluator = self._evaluators[evaluator_id]
+        except KeyError:
+            from jedi.api.environment import InterpreterEnvironment
+            evaluator = Evaluator(
+                # The project is not actually needed. Nothing should need to
+                # access it.
+                project=None,
+                environment=InterpreterEnvironment()
+            )
+            self._evaluators[evaluator_id] = evaluator
+        return evaluator
+
+    def _run(self, evaluator_id, function, args, kwargs):
+        if evaluator_id is None:
+            return function(*args, **kwargs)
+        elif function is None:
+            del self._evaluators[evaluator_id]
+        else:
+            evaluator = self._get_evaluator(function, evaluator_id)
+
+            # Exchange all handles
+            args = list(args)
+            for i, arg in enumerate(args):
+                if isinstance(arg, AccessHandle):
+                    args[i] = evaluator.compiled_subprocess.get_access_handle(arg.id)
+            for key, value in kwargs.items():
+                if isinstance(value, AccessHandle):
+                    kwargs[key] = evaluator.compiled_subprocess.get_access_handle(value.id)
+
+            return function(evaluator, *args, **kwargs)
+
+    def listen(self):
+        stdout = sys.stdout
+        # Mute stdout. Nobody should actually be able to write to it,
+        # because stdout is used for IPC.
+        sys.stdout = open(os.devnull, 'w')
+        stdin = sys.stdin
+        if sys.version_info[0] > 2:
+            stdout = stdout.buffer
+            stdin = stdin.buffer
+        # Python 2 opens streams in text mode on Windows. Set stdout and stdin
+        # to binary mode.
+        elif sys.platform == 'win32':
+            import msvcrt
+            msvcrt.setmode(stdout.fileno(), os.O_BINARY)
+            msvcrt.setmode(stdin.fileno(), os.O_BINARY)
+
+        while True:
+            try:
+                payload = pickle_load(stdin)
+            except EOFError:
+                # It looks like the parent process closed.
+                # Don't make a big fuss here and just exit.
+                exit(0)
+            try:
+                result = False, None, self._run(*payload)
+            except Exception as e:
+                result = True, traceback.format_exc(), e
+
+            pickle_dump(result, stdout, self._pickle_protocol)
+
+
+class AccessHandle(object):
+    def __init__(self, subprocess, access, id_):
+        self.access = access
+        self._subprocess = subprocess
+        self.id = id_
+
+    def add_subprocess(self, subprocess):
+        self._subprocess = subprocess
+
+    def __repr__(self):
+        try:
+            detail = self.access
+        except AttributeError:
+            detail = '#' + str(self.id)
+        return '<%s of %s>' % (self.__class__.__name__, detail)
+
+    def __getstate__(self):
+        return self.id
+
+    def __setstate__(self, state):
+        self.id = state
+
+    def __getattr__(self, name):
+        if name in ('id', 'access') or name.startswith('_'):
+            raise AttributeError("Something went wrong with unpickling")
+
+        #if not is_py3: print >> sys.stderr, name
+        #print('getattr', name, file=sys.stderr)
+        return partial(self._workaround, force_unicode(name))
+
+    def _workaround(self, name, *args, **kwargs):
+        """
+        TODO Currently we're passing slice objects around. This should not
+        happen. They are also the only unhashable objects that we're passing
+        around.
+        """
+        if args and isinstance(args[0], slice):
+            return self._subprocess.get_compiled_method_return(self.id, name, *args, **kwargs)
+        return self._cached_results(name, *args, **kwargs)
+
+    @memoize_method
+    def _cached_results(self, name, *args, **kwargs):
+        #if type(self._subprocess) == EvaluatorSubprocess:
+            #print(name, args, kwargs,
+                #self._subprocess.get_compiled_method_return(self.id, name, *args, **kwargs)
+            #)
+        return self._subprocess.get_compiled_method_return(self.id, name, *args, **kwargs)

jedi/evaluate/compiled/subprocess/__main__.py

@@ -0,0 +1,55 @@
+import os
+import sys
+
+
+def _get_paths():
+    # Get the path to jedi.
+    _d = os.path.dirname
+    _jedi_path = _d(_d(_d(_d(_d(__file__)))))
+    _parso_path = sys.argv[1]
+    # The paths are the directory that jedi and parso lie in.
+    return {'jedi': _jedi_path, 'parso': _parso_path}
+
+
+# Remove the first entry, because it's simply a directory entry that equals
+# this directory.
+del sys.path[0]
+
+if sys.version_info > (3, 4):
+    from importlib.machinery import PathFinder
+
+    class _ExactImporter(object):
+        def __init__(self, path_dct):
+            self._path_dct = path_dct
+
+        def find_module(self, fullname, path=None):
+            if path is None and fullname in self._path_dct:
+                p = self._path_dct[fullname]
+                loader = PathFinder.find_module(fullname, path=[p])
+                return loader
+            return None
+
+    # Try to import jedi/parso.
+    sys.meta_path.insert(0, _ExactImporter(_get_paths()))
+    from jedi.evaluate.compiled import subprocess  # NOQA
+    sys.meta_path.pop(0)
+else:
+    import imp
+
+    def load(name):
+        paths = list(_get_paths().values())
+        fp, pathname, description = imp.find_module(name, paths)
+        return imp.load_module(name, fp, pathname, description)
+
+    load('parso')
+    load('jedi')
+    from jedi.evaluate.compiled import subprocess  # NOQA
+
+from jedi._compatibility import highest_pickle_protocol  # noqa: E402
+
+
+# Retrieve the pickle protocol.
+host_sys_version = [int(x) for x in sys.argv[2].split('.')]
+pickle_protocol = highest_pickle_protocol([sys.version_info, host_sys_version])
+# And finally start the client.
+subprocess.Listener(pickle_protocol=pickle_protocol).listen()

jedi/evaluate/compiled/subprocess/functions.py

@@ -0,0 +1,86 @@
+from __future__ import print_function
+import sys
+import os
+
+from jedi._compatibility import find_module, cast_path, force_unicode, \
+    iter_modules, all_suffixes
+from jedi.evaluate.compiled import access
+from jedi import parser_utils
+
+
+def get_sys_path():
+    return list(map(cast_path, sys.path))
+
+
+def load_module(evaluator, **kwargs):
+    return access.load_module(evaluator, **kwargs)
+
+
+def get_compiled_method_return(evaluator, id, attribute, *args, **kwargs):
+    handle = evaluator.compiled_subprocess.get_access_handle(id)
+    return getattr(handle.access, attribute)(*args, **kwargs)
+
+
+def create_simple_object(evaluator, obj):
+    return access.create_access_path(evaluator, obj)
+
+
+def get_module_info(evaluator, sys_path=None, full_name=None, **kwargs):
+    """
+    Returns Tuple[Union[NamespaceInfo, FileIO, None], Optional[bool]]
+    """
+    if sys_path is not None:
+        sys.path, temp = sys_path, sys.path
+    try:
+        return find_module(full_name=full_name, **kwargs)
+    except ImportError:
+        return None, None
+    finally:
+        if sys_path is not None:
+            sys.path = temp
+
+
+def list_module_names(evaluator, search_path):
+    return [
+        force_unicode(name)
+        for module_loader, name, is_pkg in iter_modules(search_path)
+    ]
+
+
+def get_builtin_module_names(evaluator):
+    return list(map(force_unicode, sys.builtin_module_names))
+
+
+def _test_raise_error(evaluator, exception_type):
+    """
+    Raise an error to simulate certain problems for unit tests.
+    """
+    raise exception_type
+
+
+def _test_print(evaluator, stderr=None, stdout=None):
+    """
+    Force some prints in the subprocesses. This exists for unit tests.
+    """
+    if stderr is not None:
+        print(stderr, file=sys.stderr)
+        sys.stderr.flush()
+    if stdout is not None:
+        print(stdout)
+        sys.stdout.flush()
+
+
+def _get_init_path(directory_path):
+    """
+    The __init__ file can be searched in a directory. If found return it, else
+    None.
+    """
+    for suffix in all_suffixes():
+        path = os.path.join(directory_path, '__init__' + suffix)
+        if os.path.exists(path):
+            return path
+    return None
+
+
+def safe_literal_eval(evaluator, value):
+    return parser_utils.safe_literal_eval(value)

jedi/evaluate/context.py

@@ -1,143 +0,0 @@
-from jedi._compatibility import Python3Method
-from jedi.common import unite
-
-
-class Context(object):
-    api_type = None
-    """
-    To be defined by subclasses.
-    """
-    predefined_names = {}
-    tree_node = None
-
-    def __init__(self, evaluator, parent_context=None):
-        self.evaluator = evaluator
-        self.parent_context = parent_context
-
-    def get_parent_flow_context(self):
-        return self.parent_context
-
-    def get_root_context(self):
-        context = self
-        while True:
-            if context.parent_context is None:
-                return context
-            context = context.parent_context
-
-    def execute(self, arguments):
-        return self.evaluator.execute(self, arguments)
-
-    def execute_evaluated(self, *value_list):
-        """
-        Execute a function with already executed arguments.
-        """
-        from jedi.evaluate.param import ValuesArguments
-        arguments = ValuesArguments([[value] for value in value_list])
-        return self.execute(arguments)
-
-    def eval_node(self, node):
-        return self.evaluator.eval_element(self, node)
-
-    def eval_stmt(self, stmt, seek_name=None):
-        return self.evaluator.eval_statement(self, stmt, seek_name)
-
-    @Python3Method
-    def eval_trailer(self, types, trailer):
-        return self.evaluator.eval_trailer(self, types, trailer)
-
-    @Python3Method
-    def py__getattribute__(self, name_or_str, name_context=None, position=None,
-                           search_global=False, is_goto=False):
-        if name_context is None:
-            name_context = self
-        return self.evaluator.find_types(
-            self, name_or_str, name_context, position, search_global, is_goto)
-
-    def create_context(self, node, node_is_context=False, node_is_object=False):
-        return self.evaluator.create_context(self, node, node_is_context, node_is_object)
-
-    def is_class(self):
-        return False
-
-    def py__bool__(self):
-        """
-        Since Wrapper is a super class for classes, functions and modules,
-        the return value will always be true.
-        """
-        return True
-
-
-class TreeContext(Context):
-    def __init__(self, evaluator, parent_context=None):
-        super(TreeContext, self).__init__(evaluator, parent_context)
-        self.predefined_names = {}
-
-    def __repr__(self):
-        return '<%s: %s>' % (self.__class__.__name__, self.tree_node)
-
-
-class FlowContext(TreeContext):
-    def get_parent_flow_context(self):
-        if 1:
-            return self.parent_context
-
-
-class AbstractLazyContext(object):
-    def __init__(self, data):
-        self.data = data
-
-    def __repr__(self):
-        return '<%s: %s>' % (self.__class__.__name__, self.data)
-
-    def infer(self):
-        raise NotImplementedError
-
-
-class LazyKnownContext(AbstractLazyContext):
-    """data is a context."""
-    def infer(self):
-        return set([self.data])
-
-
-class LazyKnownContexts(AbstractLazyContext):
-    """data is a set of contexts."""
-    def infer(self):
-        return self.data
-
-
-class LazyUnknownContext(AbstractLazyContext):
-    def __init__(self):
-        super(LazyUnknownContext, self).__init__(None)
-
-    def infer(self):
-        return set()
-
-
-class LazyTreeContext(AbstractLazyContext):
-    def __init__(self, context, node):
-        super(LazyTreeContext, self).__init__(node)
-        self._context = context
-        # We need to save the predefined names. It's an unfortunate side effect
-        # that needs to be tracked otherwise results will be wrong.
-        self._predefined_names = dict(context.predefined_names)
-
-    def infer(self):
-        old, self._context.predefined_names = \
-            self._context.predefined_names, self._predefined_names
-        try:
-            return self._context.eval_node(self.data)
-        finally:
-            self._context.predefined_names = old
-
-
-def get_merged_lazy_context(lazy_contexts):
-    if len(lazy_contexts) > 1:
-        return MergedLazyContexts(lazy_contexts)
-    else:
-        return lazy_contexts[0]
-
-
-class MergedLazyContexts(AbstractLazyContext):
-    """data is a list of lazy contexts."""
-    def infer(self):
-        return unite(l.infer() for l in self.data)

jedi/evaluate/context/__init__.py

@@ -0,0 +1,6 @@
+from jedi.evaluate.context.module import ModuleContext
+from jedi.evaluate.context.klass import ClassContext
+from jedi.evaluate.context.function import FunctionContext, \
+    MethodContext, FunctionExecutionContext
+from jedi.evaluate.context.instance import AnonymousInstance, BoundMethod, \
+    CompiledInstance, AbstractInstanceContext, TreeInstance

jedi/evaluate/context/decorator.py

@@ -0,0 +1,15 @@
+'''
+Decorators are not really contexts, however we need some wrappers to improve
+docstrings and other things around decorators.
+'''
+
+from jedi.evaluate.base_context import ContextWrapper
+
+
+class Decoratee(ContextWrapper):
+    def __init__(self, wrapped_context, original_context):
+        self._wrapped_context = wrapped_context
+        self._original_context = original_context
+
+    def py__doc__(self):
+        return self._original_context.py__doc__()

jedi/evaluate/context/function.py

@@ -0,0 +1,444 @@
+from parso.python import tree
+
+from jedi._compatibility import use_metaclass
+from jedi import debug
+from jedi.evaluate.cache import evaluator_method_cache, CachedMetaClass
+from jedi.evaluate import compiled
+from jedi.evaluate import recursion
+from jedi.evaluate import docstrings
+from jedi.evaluate import flow_analysis
+from jedi.evaluate import helpers
+from jedi.evaluate.signature import TreeSignature
+from jedi.evaluate.arguments import AnonymousArguments
+from jedi.evaluate.filters import ParserTreeFilter, FunctionExecutionFilter
+from jedi.evaluate.names import ContextName, AbstractNameDefinition, ParamName
+from jedi.evaluate.base_context import ContextualizedNode, NO_CONTEXTS, \
+    ContextSet, TreeContext, ContextWrapper
+from jedi.evaluate.lazy_context import LazyKnownContexts, LazyKnownContext, \
+    LazyTreeContext
+from jedi.evaluate.context import iterable
+from jedi import parser_utils
+from jedi.evaluate.parser_cache import get_yield_exprs
+from jedi.evaluate.helpers import contexts_from_qualified_names
+
+
+class LambdaName(AbstractNameDefinition):
+    string_name = '<lambda>'
+    api_type = u'function'
+
+    def __init__(self, lambda_context):
+        self._lambda_context = lambda_context
+        self.parent_context = lambda_context.parent_context
+
+    @property
+    def start_pos(self):
+        return self._lambda_context.tree_node.start_pos
+
+    def infer(self):
+        return ContextSet([self._lambda_context])
+
+
+class FunctionAndClassBase(TreeContext):
+    def get_qualified_names(self):
+        if self.parent_context.is_class():
+            n = self.parent_context.get_qualified_names()
+            if n is None:
+                # This means that the parent class lives within a function.
+                return None
+            return n + (self.py__name__(),)
+        elif self.parent_context.is_module():
+            return (self.py__name__(),)
+        else:
+            return None
+
+
+class FunctionMixin(object):
+    api_type = u'function'
+
+    def get_filters(self, search_global=False, until_position=None, origin_scope=None):
+        if search_global:
+            yield ParserTreeFilter(
+                self.evaluator,
+                context=self,
+                until_position=until_position,
+                origin_scope=origin_scope
+            )
+        else:
+            cls = self.py__class__()
+            for instance in cls.execute_evaluated():
+                for filter in instance.get_filters(search_global=False, origin_scope=origin_scope):
+                    yield filter
+
+    def py__get__(self, instance, class_context):
+        from jedi.evaluate.context.instance import BoundMethod
+        if instance is None:
+            # Calling the Foo.bar results in the original bar function.
+            return ContextSet([self])
+        return ContextSet([BoundMethod(instance, self)])
+
+    def get_param_names(self):
+        function_execution = self.get_function_execution()
+        return [ParamName(function_execution, param.name)
+                for param in self.tree_node.get_params()]
+
+    @property
+    def name(self):
+        if self.tree_node.type == 'lambdef':
+            return LambdaName(self)
+        return ContextName(self, self.tree_node.name)
+
+    def py__name__(self):
+        return self.name.string_name
+
+    def py__call__(self, arguments):
+        function_execution = self.get_function_execution(arguments)
+        return function_execution.infer()
+
+    def get_function_execution(self, arguments=None):
+        if arguments is None:
+            arguments = AnonymousArguments()
+
+        return FunctionExecutionContext(self.evaluator, self.parent_context, self, arguments)
+
+    def get_signatures(self):
+        return [TreeSignature(f) for f in self.get_signature_functions()]
+
+
+class FunctionContext(use_metaclass(CachedMetaClass, FunctionMixin, FunctionAndClassBase)):
+    """
+    Needed because of decorators. Decorators are evaluated here.
+    """
+    def is_function(self):
+        return True
+
+    @classmethod
+    def from_context(cls, context, tree_node):
+        def create(tree_node):
+            if context.is_class():
+                return MethodContext(
+                    context.evaluator,
+                    context,
+                    parent_context=parent_context,
+                    tree_node=tree_node
+                )
+            else:
+                return cls(
+                    context.evaluator,
+                    parent_context=parent_context,
+                    tree_node=tree_node
+                )
+
+        overloaded_funcs = list(_find_overload_functions(context, tree_node))
+
+        parent_context = context
+        while parent_context.is_class() or parent_context.is_instance():
+            parent_context = parent_context.parent_context
+
+        function = create(tree_node)
+
+        if overloaded_funcs:
+            return OverloadedFunctionContext(
+                function,
+                [create(f) for f in overloaded_funcs]
+            )
+        return function
+
+    def py__class__(self):
+        c, = contexts_from_qualified_names(self.evaluator, u'types', u'FunctionType')
+        return c
+
+    def get_default_param_context(self):
+        return self.parent_context
+
+    def get_signature_functions(self):
+        return [self]
+
+
+class MethodContext(FunctionContext):
+    def __init__(self, evaluator, class_context, *args, **kwargs):
+        super(MethodContext, self).__init__(evaluator, *args, **kwargs)
+        self.class_context = class_context
+
+    def get_default_param_context(self):
+        return self.class_context
+
+    def get_qualified_names(self):
+        # Need to implement this, because the parent context of a method
+        # context is not the class context but the module.
+        names = self.class_context.get_qualified_names()
+        if names is None:
+            return None
+        return names + (self.py__name__(),)
+
+
+class FunctionExecutionContext(TreeContext):
+    """
+    This class is used to evaluate functions and their returns.
+
+    This is the most complicated class, because it contains the logic to
+    transfer parameters. It is even more complicated, because there may be
+    multiple calls to functions and recursion has to be avoided. But this is
+    responsibility of the decorators.
+    """
+    function_execution_filter = FunctionExecutionFilter
+
+    def __init__(self, evaluator, parent_context, function_context, var_args):
+        super(FunctionExecutionContext, self).__init__(
+            evaluator,
+            parent_context,
+            function_context.tree_node,
+        )
+        self.function_context = function_context
+        self.var_args = var_args
+
+    @evaluator_method_cache(default=NO_CONTEXTS)
+    @recursion.execution_recursion_decorator()
+    def get_return_values(self, check_yields=False):
+        funcdef = self.tree_node
+        if funcdef.type == 'lambdef':
+            return self.eval_node(funcdef.children[-1])
+
+        if check_yields:
+            context_set = NO_CONTEXTS
+            returns = get_yield_exprs(self.evaluator, funcdef)
+        else:
+            returns = funcdef.iter_return_stmts()
+            from jedi.evaluate.gradual.annotation import infer_return_types
+            context_set = infer_return_types(self)
+            if context_set:
+                # If there are annotations, prefer them over anything else.
+                # This will make it faster.
+                return context_set
+            context_set |= docstrings.infer_return_types(self.function_context)
+
+        for r in returns:
+            check = flow_analysis.reachability_check(self, funcdef, r)
+            if check is flow_analysis.UNREACHABLE:
+                debug.dbg('Return unreachable: %s', r)
+            else:
+                if check_yields:
+                    context_set |= ContextSet.from_sets(
+                        lazy_context.infer()
+                        for lazy_context in self._get_yield_lazy_context(r)
+                    )
+                else:
+                    try:
+                        children = r.children
+                    except AttributeError:
+                        ctx = compiled.builtin_from_name(self.evaluator, u'None')
+                        context_set |= ContextSet([ctx])
+                    else:
+                        context_set |= self.eval_node(children[1])
+            if check is flow_analysis.REACHABLE:
+                debug.dbg('Return reachable: %s', r)
+                break
+        return context_set
+
+    def _get_yield_lazy_context(self, yield_expr):
+        if yield_expr.type == 'keyword':
+            # `yield` just yields None.
+            ctx = compiled.builtin_from_name(self.evaluator, u'None')
+            yield LazyKnownContext(ctx)
+            return
+
+        node = yield_expr.children[1]
+        if node.type == 'yield_arg':  # It must be a yield from.
+            cn = ContextualizedNode(self, node.children[1])
+            for lazy_context in cn.infer().iterate(cn):
+                yield lazy_context
+        else:
+            yield LazyTreeContext(self, node)
+
+    @recursion.execution_recursion_decorator(default=iter([]))
+    def get_yield_lazy_contexts(self, is_async=False):
+        # TODO: if is_async, wrap yield statements in Awaitable/async_generator_asend
+        for_parents = [(y, tree.search_ancestor(y, 'for_stmt', 'funcdef',
+                                                'while_stmt', 'if_stmt'))
+                       for y in get_yield_exprs(self.evaluator, self.tree_node)]
+
+        # Calculate if the yields are placed within the same for loop.
+        yields_order = []
+        last_for_stmt = None
+        for yield_, for_stmt in for_parents:
+            # For really simple for loops we can predict the order. Otherwise
+            # we just ignore it.
+            parent = for_stmt.parent
+            if parent.type == 'suite':
+                parent = parent.parent
+            if for_stmt.type == 'for_stmt' and parent == self.tree_node \
+                    and parser_utils.for_stmt_defines_one_name(for_stmt):  # Simplicity for now.
+                if for_stmt == last_for_stmt:
+                    yields_order[-1][1].append(yield_)
+                else:
+                    yields_order.append((for_stmt, [yield_]))
+            elif for_stmt == self.tree_node:
+                yields_order.append((None, [yield_]))
+            else:
+                types = self.get_return_values(check_yields=True)
+                if types:
+                    yield LazyKnownContexts(types)
+                return
+            last_for_stmt = for_stmt
+
+        for for_stmt, yields in yields_order:
+            if for_stmt is None:
+                # No for_stmt, just normal yields.
+                for yield_ in yields:
+                    for result in self._get_yield_lazy_context(yield_):
+                        yield result
+            else:
+                input_node = for_stmt.get_testlist()
+                cn = ContextualizedNode(self, input_node)
+                ordered = cn.infer().iterate(cn)
+                ordered = list(ordered)
+                for lazy_context in ordered:
+                    dct = {str(for_stmt.children[1].value): lazy_context.infer()}
+                    with helpers.predefine_names(self, for_stmt, dct):
+                        for yield_in_same_for_stmt in yields:
+                            for result in self._get_yield_lazy_context(yield_in_same_for_stmt):
+                                yield result
+
+    def merge_yield_contexts(self, is_async=False):
+        return ContextSet.from_sets(
+            lazy_context.infer()
+            for lazy_context in self.get_yield_lazy_contexts()
+        )
+
+    def get_filters(self, search_global=False, until_position=None, origin_scope=None):
+        yield self.function_execution_filter(self.evaluator, self,
+                                             until_position=until_position,
+                                             origin_scope=origin_scope)
+
+    @evaluator_method_cache()
+    def get_executed_params_and_issues(self):
+        return self.var_args.get_executed_params_and_issues(self)
+
+    def matches_signature(self):
+        executed_params, issues = self.get_executed_params_and_issues()
+        if issues:
+            return False
+
+        matches = all(executed_param.matches_signature()
+                      for executed_param in executed_params)
+        if debug.enable_notice:
+            signature = parser_utils.get_call_signature(self.tree_node)
+            if matches:
+                debug.dbg("Overloading match: %s@%s (%s)",
+                          signature, self.tree_node.start_pos[0], self.var_args, color='BLUE')
+            else:
+                debug.dbg("Overloading no match: %s@%s (%s)",
+                          signature, self.tree_node.start_pos[0], self.var_args, color='BLUE')
+        return matches
+
+    def infer(self):
+        """
+        Created to be used by inheritance.
+        """
+        evaluator = self.evaluator
+        is_coroutine = self.tree_node.parent.type in ('async_stmt', 'async_funcdef')
+        is_generator = bool(get_yield_exprs(evaluator, self.tree_node))
+        from jedi.evaluate.gradual.typing import GenericClass
+
+        if is_coroutine:
+            if is_generator:
+                if evaluator.environment.version_info < (3, 6):
+                    return NO_CONTEXTS
+                async_generator_classes = evaluator.typing_module \
+                    .py__getattribute__('AsyncGenerator')
+
+                yield_contexts = self.merge_yield_contexts(is_async=True)
+                # The contravariant doesn't seem to be defined.
+                generics = (yield_contexts.py__class__(), NO_CONTEXTS)
+                return ContextSet(
+                    # In Python 3.6 AsyncGenerator is still a class.
+                    GenericClass(c, generics)
+                    for c in async_generator_classes
+                ).execute_annotation()
+            else:
+                if evaluator.environment.version_info < (3, 5):
+                    return NO_CONTEXTS
+                async_classes = evaluator.typing_module.py__getattribute__('Coroutine')
+                return_contexts = self.get_return_values()
+                # Only the first generic is relevant.
+                generics = (return_contexts.py__class__(), NO_CONTEXTS, NO_CONTEXTS)
+                return ContextSet(
+                    GenericClass(c, generics) for c in async_classes
+                ).execute_annotation()
+        else:
+            if is_generator:
+                return ContextSet([iterable.Generator(evaluator, self)])
+            else:
+                return self.get_return_values()
+
+
+class OverloadedFunctionContext(FunctionMixin, ContextWrapper):
+    def __init__(self, function, overloaded_functions):
+        super(OverloadedFunctionContext, self).__init__(function)
+        self._overloaded_functions = overloaded_functions
+
+    def py__call__(self, arguments):
+        debug.dbg("Execute overloaded function %s", self._wrapped_context, color='BLUE')
+        function_executions = []
+        context_set = NO_CONTEXTS
+        matched = False
+        for f in self._overloaded_functions:
+            function_execution = f.get_function_execution(arguments)
+            function_executions.append(function_execution)
+            if function_execution.matches_signature():
+                matched = True
+                return function_execution.infer()
+
+        if matched:
+            return context_set
+
+        if self.evaluator.is_analysis:
+            # In this case we want precision.
+            return NO_CONTEXTS
+        return ContextSet.from_sets(fe.infer() for fe in function_executions)
+
+    def get_signature_functions(self):
+        return self._overloaded_functions
+
+
+def _find_overload_functions(context, tree_node):
+    def _is_overload_decorated(funcdef):
+        if funcdef.parent.type == 'decorated':
+            decorators = funcdef.parent.children[0]
+            if decorators.type == 'decorator':
+                decorators = [decorators]
+            else:
+                decorators = decorators.children
+            for decorator in decorators:
+                dotted_name = decorator.children[1]
+                if dotted_name.type == 'name' and dotted_name.value == 'overload':
+                    # TODO check with contexts if it's the right overload
+                    return True
+        return False
+
+    if tree_node.type == 'lambdef':
+        return
+
+    if _is_overload_decorated(tree_node):
+        yield tree_node
+
+    while True:
+        filter = ParserTreeFilter(
+            context.evaluator,
+            context,
+            until_position=tree_node.start_pos
+        )
+        names = filter.get(tree_node.name.value)
+        assert isinstance(names, list)
+        if not names:
+            break
+
+        found = False
+        for name in names:
+            funcdef = name.tree_name.parent
+            if funcdef.type == 'funcdef' and _is_overload_decorated(funcdef):
+                tree_node = funcdef
+                found = True
+                yield funcdef
+
+        if not found:
+            break

jedi/evaluate/context/instance.py

@@ -0,0 +1,531 @@
+from abc import abstractproperty
+
+from jedi import debug
+from jedi import settings
+from jedi.evaluate import compiled
+from jedi.evaluate.compiled.context import CompiledObjectFilter
+from jedi.evaluate.helpers import contexts_from_qualified_names
+from jedi.evaluate.filters import AbstractFilter
+from jedi.evaluate.names import ContextName, TreeNameDefinition
+from jedi.evaluate.base_context import Context, NO_CONTEXTS, ContextSet, \
+    iterator_to_context_set, ContextWrapper
+from jedi.evaluate.lazy_context import LazyKnownContext, LazyKnownContexts
+from jedi.evaluate.cache import evaluator_method_cache
+from jedi.evaluate.arguments import AnonymousArguments, \
+    ValuesArguments, TreeArgumentsWrapper
+from jedi.evaluate.context.function import \
+    FunctionContext, FunctionMixin, OverloadedFunctionContext
+from jedi.evaluate.context.klass import ClassContext, apply_py__get__, \
+    ClassFilter
+from jedi.evaluate.context import iterable
+from jedi.parser_utils import get_parent_scope
+
+
+class InstanceExecutedParam(object):
+    def __init__(self, instance, tree_param):
+        self._instance = instance
+        self._tree_param = tree_param
+        self.string_name = self._tree_param.name.value
+
+    def infer(self):
+        return ContextSet([self._instance])
+
+    def matches_signature(self):
+        return True
+
+
+class AnonymousInstanceArguments(AnonymousArguments):
+    def __init__(self, instance):
+        self._instance = instance
+
+    def get_executed_params_and_issues(self, execution_context):
+        from jedi.evaluate.dynamic import search_params
+        tree_params = execution_context.tree_node.get_params()
+        if not tree_params:
+            return [], []
+
+        self_param = InstanceExecutedParam(self._instance, tree_params[0])
+        if len(tree_params) == 1:
+            # If the only param is self, we don't need to try to find
+            # executions of this function, we have all the params already.
+            return [self_param], []
+        executed_params = list(search_params(
+            execution_context.evaluator,
+            execution_context,
+            execution_context.tree_node
+        ))
+        executed_params[0] = self_param
+        return executed_params, []
+
+
+class AbstractInstanceContext(Context):
+    """
+    This class is used to evaluate instances.
+    """
+    api_type = u'instance'
+
+    def __init__(self, evaluator, parent_context, class_context, var_args):
+        super(AbstractInstanceContext, self).__init__(evaluator, parent_context)
+        # Generated instances are classes that are just generated by self
+        # (No var_args) used.
+        self.class_context = class_context
+        self.var_args = var_args
+
+    def is_instance(self):
+        return True
+
+    def get_qualified_names(self):
+        return self.class_context.get_qualified_names()
+
+    def get_annotated_class_object(self):
+        return self.class_context  # This is the default.
+
+    def py__call__(self, arguments):
+        names = self.get_function_slot_names(u'__call__')
+        if not names:
+            # Means the Instance is not callable.
+            return super(AbstractInstanceContext, self).py__call__(arguments)
+
+        return ContextSet.from_sets(name.infer().execute(arguments) for name in names)
+
+    def py__class__(self):
+        return self.class_context
+
+    def py__bool__(self):
+        # Signalize that we don't know about the bool type.
+        return None
+
+    def get_function_slot_names(self, name):
+        # Python classes don't look at the dictionary of the instance when
+        # looking up `__call__`. This is something that has to do with Python's
+        # internal slot system (note: not __slots__, but C slots).
+        for filter in self.get_filters(include_self_names=False):
+            names = filter.get(name)
+            if names:
+                return names
+        return []
+
+    def execute_function_slots(self, names, *evaluated_args):
+        return ContextSet.from_sets(
+            name.infer().execute_evaluated(*evaluated_args)
+            for name in names
+        )
+
+    def py__get__(self, obj, class_context):
+        """
+        obj may be None.
+        """
+        # Arguments in __get__ descriptors are obj, class.
+        # `method` is the new parent of the array, don't know if that's good.
+        names = self.get_function_slot_names(u'__get__')
+        if names:
+            if obj is None:
+                obj = compiled.builtin_from_name(self.evaluator, u'None')
+            return self.execute_function_slots(names, obj, class_context)
+        else:
+            return ContextSet([self])
+
+    def get_filters(self, search_global=None, until_position=None,
+                    origin_scope=None, include_self_names=True):
+        class_context = self.get_annotated_class_object()
+        if include_self_names:
+            for cls in class_context.py__mro__():
+                if not isinstance(cls, compiled.CompiledObject) \
+                        or cls.tree_node is not None:
+                    # In this case we're excluding compiled objects that are
+                    # not fake objects. It doesn't make sense for normal
+                    # compiled objects to search for self variables.
+                    yield SelfAttributeFilter(self.evaluator, self, cls, origin_scope)
+
+        class_filters = class_context.get_filters(
+            search_global=False,
+            origin_scope=origin_scope,
+            is_instance=True,
+        )
+        for f in class_filters:
+            if isinstance(f, ClassFilter):
+                yield InstanceClassFilter(self.evaluator, self, f)
+            elif isinstance(f, CompiledObjectFilter):
+                yield CompiledInstanceClassFilter(self.evaluator, self, f)
+            else:
+                # Propably from the metaclass.
+                yield f
+
+    def py__getitem__(self, index_context_set, contextualized_node):
+        names = self.get_function_slot_names(u'__getitem__')
+        if not names:
+            return super(AbstractInstanceContext, self).py__getitem__(
+                index_context_set,
+                contextualized_node,
+            )
+
+        args = ValuesArguments([index_context_set])
+        return ContextSet.from_sets(name.infer().execute(args) for name in names)
+
+    def py__iter__(self, contextualized_node=None):
+        iter_slot_names = self.get_function_slot_names(u'__iter__')
+        if not iter_slot_names:
+            return super(AbstractInstanceContext, self).py__iter__(contextualized_node)
+
+        def iterate():
+            for generator in self.execute_function_slots(iter_slot_names):
+                if generator.is_instance() and not generator.is_compiled():
+                    # `__next__` logic.
+                    if self.evaluator.environment.version_info.major == 2:
+                        name = u'next'
+                    else:
+                        name = u'__next__'
+                    next_slot_names = generator.get_function_slot_names(name)
+                    if next_slot_names:
+                        yield LazyKnownContexts(
+                            generator.execute_function_slots(next_slot_names)
+                        )
+                    else:
+                        debug.warning('Instance has no __next__ function in %s.', generator)
+                else:
+                    for lazy_context in generator.py__iter__():
+                        yield lazy_context
+        return iterate()
+
+    @abstractproperty
+    def name(self):
+        pass
+
+    def create_init_executions(self):
+        for name in self.get_function_slot_names(u'__init__'):
+            # TODO is this correct? I think we need to check for functions.
+            if isinstance(name, LazyInstanceClassName):
+                function = FunctionContext.from_context(
+                    self.parent_context,
+                    name.tree_name.parent
+                )
+                bound_method = BoundMethod(self, function)
+                yield bound_method.get_function_execution(self.var_args)
+
+    @evaluator_method_cache()
+    def create_instance_context(self, class_context, node):
+        if node.parent.type in ('funcdef', 'classdef'):
+            node = node.parent
+        scope = get_parent_scope(node)
+        if scope == class_context.tree_node:
+            return class_context
+        else:
+            parent_context = self.create_instance_context(class_context, scope)
+            if scope.type == 'funcdef':
+                func = FunctionContext.from_context(
+                    parent_context,
+                    scope,
+                )
+                bound_method = BoundMethod(self, func)
+                if scope.name.value == '__init__' and parent_context == class_context:
+                    return bound_method.get_function_execution(self.var_args)
+                else:
+                    return bound_method.get_function_execution()
+            elif scope.type == 'classdef':
+                class_context = ClassContext(self.evaluator, parent_context, scope)
+                return class_context
+            elif scope.type in ('comp_for', 'sync_comp_for'):
+                # Comprehensions currently don't have a special scope in Jedi.
+                return self.create_instance_context(class_context, scope)
+            else:
+                raise NotImplementedError
+        return class_context
+
+    def get_signatures(self):
+        call_funcs = self.py__getattribute__('__call__').py__get__(self, self.class_context)
+        return [s.bind(self) for s in call_funcs.get_signatures()]
+
+    def __repr__(self):
+        return "<%s of %s(%s)>" % (self.__class__.__name__, self.class_context,
+                                   self.var_args)
+
+
+class CompiledInstance(AbstractInstanceContext):
+    def __init__(self, evaluator, parent_context, class_context, var_args):
+        self._original_var_args = var_args
+        super(CompiledInstance, self).__init__(evaluator, parent_context, class_context, var_args)
+
+    @property
+    def name(self):
+        return compiled.CompiledContextName(self, self.class_context.name.string_name)
+
+    def get_first_non_keyword_argument_contexts(self):
+        key, lazy_context = next(self._original_var_args.unpack(), ('', None))
+        if key is not None:
+            return NO_CONTEXTS
+
+        return lazy_context.infer()
+
+    def is_stub(self):
+        return False
+
+
+class TreeInstance(AbstractInstanceContext):
+    def __init__(self, evaluator, parent_context, class_context, var_args):
+        # I don't think that dynamic append lookups should happen here. That
+        # sounds more like something that should go to py__iter__.
+        if class_context.py__name__() in ['list', 'set'] \
+                and parent_context.get_root_context() == evaluator.builtins_module:
+            # compare the module path with the builtin name.
+            if settings.dynamic_array_additions:
+                var_args = iterable.get_dynamic_array_instance(self, var_args)
+
+        super(TreeInstance, self).__init__(evaluator, parent_context,
+                                           class_context, var_args)
+        self.tree_node = class_context.tree_node
+
+    @property
+    def name(self):
+        return ContextName(self, self.class_context.name.tree_name)
+
+    # This can recurse, if the initialization of the class includes a reference
+    # to itself.
+    @evaluator_method_cache(default=None)
+    def _get_annotated_class_object(self):
+        from jedi.evaluate.gradual.annotation import py__annotations__, \
+            infer_type_vars_for_execution
+
+        for func in self._get_annotation_init_functions():
+            # Just take the first result, it should always be one, because we
+            # control the typeshed code.
+            bound = BoundMethod(self, func)
+            execution = bound.get_function_execution(self.var_args)
+            if not execution.matches_signature():
+                # First check if the signature even matches, if not we don't
+                # need to infer anything.
+                continue
+
+            all_annotations = py__annotations__(execution.tree_node)
+            defined, = self.class_context.define_generics(
+                infer_type_vars_for_execution(execution, all_annotations),
+            )
+            debug.dbg('Inferred instance context as %s', defined, color='BLUE')
+            return defined
+        return None
+
+    def get_annotated_class_object(self):
+        return self._get_annotated_class_object() or self.class_context
+
+    def _get_annotation_init_functions(self):
+        filter = next(self.class_context.get_filters())
+        for init_name in filter.get('__init__'):
+            for init in init_name.infer():
+                if init.is_function():
+                    for signature in init.get_signatures():
+                        yield signature.context
+
+
+class AnonymousInstance(TreeInstance):
+    def __init__(self, evaluator, parent_context, class_context):
+        super(AnonymousInstance, self).__init__(
+            evaluator,
+            parent_context,
+            class_context,
+            var_args=AnonymousInstanceArguments(self),
+        )
+
+    def get_annotated_class_object(self):
+        return self.class_context  # This is the default.
+
+
+class CompiledInstanceName(compiled.CompiledName):
+
+    def __init__(self, evaluator, instance, klass, name):
+        super(CompiledInstanceName, self).__init__(
+            evaluator,
+            klass.parent_context,
+            name.string_name
+        )
+        self._instance = instance
+        self._class_member_name = name
+
+    @iterator_to_context_set
+    def infer(self):
+        for result_context in self._class_member_name.infer():
+            if result_context.api_type == 'function':
+                yield CompiledBoundMethod(result_context)
+            else:
+                yield result_context
+
+
+class CompiledInstanceClassFilter(AbstractFilter):
+    name_class = CompiledInstanceName
+
+    def __init__(self, evaluator, instance, f):
+        self._evaluator = evaluator
+        self._instance = instance
+        self._class_filter = f
+
+    def get(self, name):
+        return self._convert(self._class_filter.get(name))
+
+    def values(self):
+        return self._convert(self._class_filter.values())
+
+    def _convert(self, names):
+        klass = self._class_filter.compiled_object
+        return [
+            CompiledInstanceName(self._evaluator, self._instance, klass, n)
+            for n in names
+        ]
+
+
+class BoundMethod(FunctionMixin, ContextWrapper):
+    def __init__(self, instance, function):
+        super(BoundMethod, self).__init__(function)
+        self.instance = instance
+
+    def is_bound_method(self):
+        return True
+
+    def py__class__(self):
+        c, = contexts_from_qualified_names(self.evaluator, u'types', u'MethodType')
+        return c
+
+    def _get_arguments(self, arguments):
+        if arguments is None:
+            arguments = AnonymousInstanceArguments(self.instance)
+
+        return InstanceArguments(self.instance, arguments)
+
+    def get_function_execution(self, arguments=None):
+        arguments = self._get_arguments(arguments)
+        return super(BoundMethod, self).get_function_execution(arguments)
+
+    def py__call__(self, arguments):
+        if isinstance(self._wrapped_context, OverloadedFunctionContext):
+            return self._wrapped_context.py__call__(self._get_arguments(arguments))
+
+        function_execution = self.get_function_execution(arguments)
+        return function_execution.infer()
+
+    def get_signature_functions(self):
+        return [
+            BoundMethod(self.instance, f)
+            for f in self._wrapped_context.get_signature_functions()
+        ]
+
+    def get_signatures(self):
+        return [sig.bind(self) for sig in super(BoundMethod, self).get_signatures()]
+
+    def __repr__(self):
+        return '<%s: %s>' % (self.__class__.__name__, self._wrapped_context)
+
+
+class CompiledBoundMethod(ContextWrapper):
+    def is_bound_method(self):
+        return True
+
+    def get_signatures(self):
+        return [sig.bind(self) for sig in self._wrapped_context.get_signatures()]
+
+
+class SelfName(TreeNameDefinition):
+    """
+    This name calculates the parent_context lazily.
+    """
+    def __init__(self, instance, class_context, tree_name):
+        self._instance = instance
+        self.class_context = class_context
+        self.tree_name = tree_name
+
+    @property
+    def parent_context(self):
+        return self._instance.create_instance_context(self.class_context, self.tree_name)
+
+
+class LazyInstanceClassName(object):
+    def __init__(self, instance, class_context, class_member_name):
+        self._instance = instance
+        self.class_context = class_context
+        self._class_member_name = class_member_name
+
+    @iterator_to_context_set
+    def infer(self):
+        for result_context in self._class_member_name.infer():
+            for c in apply_py__get__(result_context, self._instance, self.class_context):
+                yield c
+
+    def __getattr__(self, name):
+        return getattr(self._class_member_name, name)
+
+    def __repr__(self):
+        return '<%s: %s>' % (self.__class__.__name__, self._class_member_name)
+
+
+class InstanceClassFilter(AbstractFilter):
+    """
+    This filter is special in that it uses the class filter and wraps the
+    resulting names in LazyINstanceClassName. The idea is that the class name
+    filtering can be very flexible and always be reflected in instances.
+    """
+    def __init__(self, evaluator, instance, class_filter):
+        self._instance = instance
+        self._class_filter = class_filter
+
+    def get(self, name):
+        return self._convert(self._class_filter.get(name, from_instance=True))
+
+    def values(self):
+        return self._convert(self._class_filter.values(from_instance=True))
+
+    def _convert(self, names):
+        return [LazyInstanceClassName(self._instance, self._class_filter.context, n) for n in names]
+
+    def __repr__(self):
+        return '<%s for %s>' % (self.__class__.__name__, self._class_filter.context)
+
+
+class SelfAttributeFilter(ClassFilter):
+    """
+    This class basically filters all the use cases where `self.*` was assigned.
+    """
+    name_class = SelfName
+
+    def __init__(self, evaluator, context, class_context, origin_scope):
+        super(SelfAttributeFilter, self).__init__(
+            evaluator=evaluator,
+            context=context,
+            node_context=class_context,
+            origin_scope=origin_scope,
+            is_instance=True,
+        )
+        self._class_context = class_context
+
+    def _filter(self, names):
+        names = self._filter_self_names(names)
+        start, end = self._parser_scope.start_pos, self._parser_scope.end_pos
+        return [n for n in names if start < n.start_pos < end]
+
+    def _filter_self_names(self, names):
+        for name in names:
+            trailer = name.parent
+            if trailer.type == 'trailer' \
+                    and len(trailer.parent.children) == 2 \
+                    and trailer.children[0] == '.':
+                if name.is_definition() and self._access_possible(name, from_instance=True):
+                    # TODO filter non-self assignments.
+                    yield name
+
+    def _convert_names(self, names):
+        return [self.name_class(self.context, self._class_context, name) for name in names]
+
+    def _check_flows(self, names):
+        return names
+
+
+class InstanceArguments(TreeArgumentsWrapper):
+    def __init__(self, instance, arguments):
+        super(InstanceArguments, self).__init__(arguments)
+        self.instance = instance
+
+    def unpack(self, func=None):
+        yield None, LazyKnownContext(self.instance)
+        for values in self._wrapped_arguments.unpack(func):
+            yield values
+
+    def get_executed_params_and_issues(self, execution_context):
+        if isinstance(self._wrapped_arguments, AnonymousInstanceArguments):
+            return self._wrapped_arguments.get_executed_params_and_issues(execution_context)
+
+        return super(InstanceArguments, self).get_executed_params_and_issues(execution_context)

jedi/evaluate/context/iterable.py

@@ -0,0 +1,821 @@
+"""
+Contains all classes and functions to deal with lists, dicts, generators and
+iterators in general.
+
+Array modifications
+*******************
+
+If the content of an array (``set``/``list``) is requested somewhere, the
+current module will be checked for appearances of ``arr.append``,
+``arr.insert``, etc.  If the ``arr`` name points to an actual array, the
+content will be added
+
+This can be really cpu intensive, as you can imagine. Because |jedi| has to
+follow **every** ``append`` and check wheter it's the right array. However this
+works pretty good, because in *slow* cases, the recursion detector and other
+settings will stop this process.
+
+It is important to note that:
+
+1. Array modfications work only in the current module.
+2. Jedi only checks Array additions; ``list.pop``, etc are ignored.
+"""
+import sys
+
+from jedi import debug
+from jedi import settings
+from jedi._compatibility import force_unicode, is_py3
+from jedi.evaluate import compiled
+from jedi.evaluate import analysis
+from jedi.evaluate import recursion
+from jedi.evaluate.lazy_context import LazyKnownContext, LazyKnownContexts, \
+    LazyTreeContext
+from jedi.evaluate.helpers import get_int_or_none, is_string, \
+    predefine_names, evaluate_call_of_leaf, reraise_getitem_errors, \
+    SimpleGetItemNotFound
+from jedi.evaluate.utils import safe_property, to_list
+from jedi.evaluate.cache import evaluator_method_cache
+from jedi.evaluate.filters import ParserTreeFilter, LazyAttributeOverwrite, \
+    publish_method
+from jedi.evaluate.base_context import ContextSet, Context, NO_CONTEXTS, \
+    TreeContext, ContextualizedNode, iterate_contexts, HelperContextMixin, _sentinel
+from jedi.parser_utils import get_sync_comp_fors
+
+
+class IterableMixin(object):
+    def py__stop_iteration_returns(self):
+        return ContextSet([compiled.builtin_from_name(self.evaluator, u'None')])
+
+    # At the moment, safe values are simple values like "foo", 1 and not
+    # lists/dicts. Therefore as a small speed optimization we can just do the
+    # default instead of resolving the lazy wrapped contexts, that are just
+    # doing this in the end as well.
+    # This mostly speeds up patterns like `sys.version_info >= (3, 0)` in
+    # typeshed.
+    if sys.version_info[0] == 2:
+        # Python 2...........
+        def get_safe_value(self, default=_sentinel):
+            if default is _sentinel:
+                raise ValueError("There exists no safe value for context %s" % self)
+            return default
+    else:
+        get_safe_value = Context.get_safe_value
+
+
+class GeneratorBase(LazyAttributeOverwrite, IterableMixin):
+    array_type = None
+
+    def _get_wrapped_context(self):
+        generator, = self.evaluator.typing_module \
+            .py__getattribute__('Generator') \
+            .execute_annotation()
+        return generator
+
+    def is_instance(self):
+        return False
+
+    def py__bool__(self):
+        return True
+
+    @publish_method('__iter__')
+    def py__iter__(self, contextualized_node=None):
+        return ContextSet([self])
+
+    @publish_method('send')
+    @publish_method('next', python_version_match=2)
+    @publish_method('__next__', python_version_match=3)
+    def py__next__(self):
+        return ContextSet.from_sets(lazy_context.infer() for lazy_context in self.py__iter__())
+
+    def py__stop_iteration_returns(self):
+        return ContextSet([compiled.builtin_from_name(self.evaluator, u'None')])
+
+    @property
+    def name(self):
+        return compiled.CompiledContextName(self, 'Generator')
+
+
+class Generator(GeneratorBase):
+    """Handling of `yield` functions."""
+    def __init__(self, evaluator, func_execution_context):
+        super(Generator, self).__init__(evaluator)
+        self._func_execution_context = func_execution_context
+
+    def py__iter__(self, contextualized_node=None):
+        return self._func_execution_context.get_yield_lazy_contexts()
+
+    def py__stop_iteration_returns(self):
+        return self._func_execution_context.get_return_values()
+
+    def __repr__(self):
+        return "<%s of %s>" % (type(self).__name__, self._func_execution_context)
+
+
+class CompForContext(TreeContext):
+    @classmethod
+    def from_comp_for(cls, parent_context, comp_for):
+        return cls(parent_context.evaluator, parent_context, comp_for)
+
+    def get_filters(self, search_global=False, until_position=None, origin_scope=None):
+        yield ParserTreeFilter(self.evaluator, self)
+
+
+def comprehension_from_atom(evaluator, context, atom):
+    bracket = atom.children[0]
+    test_list_comp = atom.children[1]
+
+    if bracket == '{':
+        if atom.children[1].children[1] == ':':
+            sync_comp_for = test_list_comp.children[3]
+            if sync_comp_for.type == 'comp_for':
+                sync_comp_for = sync_comp_for.children[1]
+
+            return DictComprehension(
+                evaluator,
+                context,
+                sync_comp_for_node=sync_comp_for,
+                key_node=test_list_comp.children[0],
+                value_node=test_list_comp.children[2],
+            )
+        else:
+            cls = SetComprehension
+    elif bracket == '(':
+        cls = GeneratorComprehension
+    elif bracket == '[':
+        cls = ListComprehension
+
+    sync_comp_for = test_list_comp.children[1]
+    if sync_comp_for.type == 'comp_for':
+        sync_comp_for = sync_comp_for.children[1]
+
+    return cls(
+        evaluator,
+        defining_context=context,
+        sync_comp_for_node=sync_comp_for,
+        entry_node=test_list_comp.children[0],
+    )
+
+
+class ComprehensionMixin(object):
+    @evaluator_method_cache()
+    def _get_comp_for_context(self, parent_context, comp_for):
+        return CompForContext.from_comp_for(parent_context, comp_for)
+
+    def _nested(self, comp_fors, parent_context=None):
+        comp_for = comp_fors[0]
+
+        is_async = comp_for.parent.type == 'comp_for'
+
+        input_node = comp_for.children[3]
+        parent_context = parent_context or self._defining_context
+        input_types = parent_context.eval_node(input_node)
+        # TODO: simulate await if self.is_async
+
+        cn = ContextualizedNode(parent_context, input_node)
+        iterated = input_types.iterate(cn, is_async=is_async)
+        exprlist = comp_for.children[1]
+        for i, lazy_context in enumerate(iterated):
+            types = lazy_context.infer()
+            dct = unpack_tuple_to_dict(parent_context, types, exprlist)
+            context_ = self._get_comp_for_context(
+                parent_context,
+                comp_for,
+            )
+            with predefine_names(context_, comp_for, dct):
+                try:
+                    for result in self._nested(comp_fors[1:], context_):
+                        yield result
+                except IndexError:
+                    iterated = context_.eval_node(self._entry_node)
+                    if self.array_type == 'dict':
+                        yield iterated, context_.eval_node(self._value_node)
+                    else:
+                        yield iterated
+
+    @evaluator_method_cache(default=[])
+    @to_list
+    def _iterate(self):
+        comp_fors = tuple(get_sync_comp_fors(self._sync_comp_for_node))
+        for result in self._nested(comp_fors):
+            yield result
+
+    def py__iter__(self, contextualized_node=None):
+        for set_ in self._iterate():
+            yield LazyKnownContexts(set_)
+
+    def __repr__(self):
+        return "<%s of %s>" % (type(self).__name__, self._sync_comp_for_node)
+
+
+class _DictMixin(object):
+    def _get_generics(self):
+        return tuple(c_set.py__class__() for c_set in self.get_mapping_item_contexts())
+
+
+class Sequence(LazyAttributeOverwrite, IterableMixin):
+    api_type = u'instance'
+
+    @property
+    def name(self):
+        return compiled.CompiledContextName(self, self.array_type)
+
+    def _get_generics(self):
+        return (self.merge_types_of_iterate().py__class__(),)
+
+    def _get_wrapped_context(self):
+        from jedi.evaluate.gradual.typing import GenericClass
+        klass = compiled.builtin_from_name(self.evaluator, self.array_type)
+        c, = GenericClass(klass, self._get_generics()).execute_annotation()
+        return c
+
+    def py__bool__(self):
+        return None  # We don't know the length, because of appends.
+
+    def py__class__(self):
+        return compiled.builtin_from_name(self.evaluator, self.array_type)
+
+    @safe_property
+    def parent(self):
+        return self.evaluator.builtins_module
+
+    def py__getitem__(self, index_context_set, contextualized_node):
+        if self.array_type == 'dict':
+            return self._dict_values()
+        return iterate_contexts(ContextSet([self]))
+
+
+class _BaseComprehension(ComprehensionMixin):
+    def __init__(self, evaluator, defining_context, sync_comp_for_node, entry_node):
+        assert sync_comp_for_node.type == 'sync_comp_for'
+        super(_BaseComprehension, self).__init__(evaluator)
+        self._defining_context = defining_context
+        self._sync_comp_for_node = sync_comp_for_node
+        self._entry_node = entry_node
+
+
+class ListComprehension(_BaseComprehension, Sequence):
+    array_type = u'list'
+
+    def py__simple_getitem__(self, index):
+        if isinstance(index, slice):
+            return ContextSet([self])
+
+        all_types = list(self.py__iter__())
+        with reraise_getitem_errors(IndexError, TypeError):
+            lazy_context = all_types[index]
+        return lazy_context.infer()
+
+
+class SetComprehension(_BaseComprehension, Sequence):
+    array_type = u'set'
+
+
+class GeneratorComprehension(_BaseComprehension, GeneratorBase):
+    pass
+
+
+class DictComprehension(ComprehensionMixin, Sequence):
+    array_type = u'dict'
+
+    def __init__(self, evaluator, defining_context, sync_comp_for_node, key_node, value_node):
+        assert sync_comp_for_node.type == 'sync_comp_for'
+        super(DictComprehension, self).__init__(evaluator)
+        self._defining_context = defining_context
+        self._sync_comp_for_node = sync_comp_for_node
+        self._entry_node = key_node
+        self._value_node = value_node
+
+    def py__iter__(self, contextualized_node=None):
+        for keys, values in self._iterate():
+            yield LazyKnownContexts(keys)
+
+    def py__simple_getitem__(self, index):
+        for keys, values in self._iterate():
+            for k in keys:
+                if isinstance(k, compiled.CompiledObject):
+                    # Be careful in the future if refactoring, index could be a
+                    # slice.
+                    if k.get_safe_value(default=object()) == index:
+                        return values
+        raise SimpleGetItemNotFound()
+
+    def _dict_keys(self):
+        return ContextSet.from_sets(keys for keys, values in self._iterate())
+
+    def _dict_values(self):
+        return ContextSet.from_sets(values for keys, values in self._iterate())
+
+    @publish_method('values')
+    def _imitate_values(self):
+        lazy_context = LazyKnownContexts(self._dict_values())
+        return ContextSet([FakeSequence(self.evaluator, u'list', [lazy_context])])
+
+    @publish_method('items')
+    def _imitate_items(self):
+        lazy_contexts = [
+            LazyKnownContext(
+                FakeSequence(
+                    self.evaluator,
+                    u'tuple',
+                    [LazyKnownContexts(key),
+                     LazyKnownContexts(value)]
+                )
+            )
+            for key, value in self._iterate()
+        ]
+
+        return ContextSet([FakeSequence(self.evaluator, u'list', lazy_contexts)])
+
+    def get_mapping_item_contexts(self):
+        return self._dict_keys(), self._dict_values()
+
+    def exact_key_items(self):
+        # NOTE: A smarter thing can probably done here to achieve better
+        # completions, but at least like this jedi doesn't crash
+        return []
+
+
+class SequenceLiteralContext(Sequence):
+    _TUPLE_LIKE = 'testlist_star_expr', 'testlist', 'subscriptlist'
+    mapping = {'(': u'tuple',
+               '[': u'list',
+               '{': u'set'}
+
+    def __init__(self, evaluator, defining_context, atom):
+        super(SequenceLiteralContext, self).__init__(evaluator)
+        self.atom = atom
+        self._defining_context = defining_context
+
+        if self.atom.type in self._TUPLE_LIKE:
+            self.array_type = u'tuple'
+        else:
+            self.array_type = SequenceLiteralContext.mapping[atom.children[0]]
+            """The builtin name of the array (list, set, tuple or dict)."""
+
+    def py__simple_getitem__(self, index):
+        """Here the index is an int/str. Raises IndexError/KeyError."""
+        if self.array_type == u'dict':
+            compiled_obj_index = compiled.create_simple_object(self.evaluator, index)
+            for key, value in self.get_tree_entries():
+                for k in self._defining_context.eval_node(key):
+                    try:
+                        method = k.execute_operation
+                    except AttributeError:
+                        pass
+                    else:
+                        if method(compiled_obj_index, u'==').get_safe_value():
+                            return self._defining_context.eval_node(value)
+            raise SimpleGetItemNotFound('No key found in dictionary %s.' % self)
+
+        if isinstance(index, slice):
+            return ContextSet([self])
+        else:
+            with reraise_getitem_errors(TypeError, KeyError, IndexError):
+                node = self.get_tree_entries()[index]
+            return self._defining_context.eval_node(node)
+
+    def py__iter__(self, contextualized_node=None):
+        """
+        While values returns the possible values for any array field, this
+        function returns the value for a certain index.
+        """
+        if self.array_type == u'dict':
+            # Get keys.
+            types = NO_CONTEXTS
+            for k, _ in self.get_tree_entries():
+                types |= self._defining_context.eval_node(k)
+            # We don't know which dict index comes first, therefore always
+            # yield all the types.
+            for _ in types:
+                yield LazyKnownContexts(types)
+        else:
+            for node in self.get_tree_entries():
+                if node == ':' or node.type == 'subscript':
+                    # TODO this should probably use at least part of the code
+                    #      of eval_subscript_list.
+                    yield LazyKnownContext(Slice(self._defining_context, None, None, None))
+                else:
+                    yield LazyTreeContext(self._defining_context, node)
+            for addition in check_array_additions(self._defining_context, self):
+                yield addition
+
+    def py__len__(self):
+        # This function is not really used often. It's more of a try.
+        return len(self.get_tree_entries())
+
+    def _dict_values(self):
+        return ContextSet.from_sets(
+            self._defining_context.eval_node(v)
+            for k, v in self.get_tree_entries()
+        )
+
+    def get_tree_entries(self):
+        c = self.atom.children
+
+        if self.atom.type in self._TUPLE_LIKE:
+            return c[::2]
+
+        array_node = c[1]
+        if array_node in (']', '}', ')'):
+            return []  # Direct closing bracket, doesn't contain items.
+
+        if array_node.type == 'testlist_comp':
+            # filter out (for now) pep 448 single-star unpacking
+            return [value for value in array_node.children[::2]
+                    if value.type != "star_expr"]
+        elif array_node.type == 'dictorsetmaker':
+            kv = []
+            iterator = iter(array_node.children)
+            for key in iterator:
+                if key == "**":
+                    # dict with pep 448 double-star unpacking
+                    # for now ignoring the values imported by **
+                    next(iterator)
+                    next(iterator, None)  # Possible comma.
+                else:
+                    op = next(iterator, None)
+                    if op is None or op == ',':
+                        if key.type == "star_expr":
+                            # pep 448 single-star unpacking
+                            # for now ignoring values imported by *
+                            pass
+                        else:
+                            kv.append(key)  # A set.
+                    else:
+                        assert op == ':'  # A dict.
+                        kv.append((key, next(iterator)))
+                        next(iterator, None)  # Possible comma.
+            return kv
+        else:
+            if array_node.type == "star_expr":
+                # pep 448 single-star unpacking
+                # for now ignoring values imported by *
+                return []
+            else:
+                return [array_node]
+
+    def exact_key_items(self):
+        """
+        Returns a generator of tuples like dict.items(), where the key is
+        resolved (as a string) and the values are still lazy contexts.
+        """
+        for key_node, value in self.get_tree_entries():
+            for key in self._defining_context.eval_node(key_node):
+                if is_string(key):
+                    yield key.get_safe_value(), LazyTreeContext(self._defining_context, value)
+
+    def __repr__(self):
+        return "<%s of %s>" % (self.__class__.__name__, self.atom)
+
+
+class DictLiteralContext(_DictMixin, SequenceLiteralContext):
+    array_type = u'dict'
+
+    def __init__(self, evaluator, defining_context, atom):
+        super(SequenceLiteralContext, self).__init__(evaluator)
+        self._defining_context = defining_context
+        self.atom = atom
+
+    @publish_method('values')
+    def _imitate_values(self):
+        lazy_context = LazyKnownContexts(self._dict_values())
+        return ContextSet([FakeSequence(self.evaluator, u'list', [lazy_context])])
+
+    @publish_method('items')
+    def _imitate_items(self):
+        lazy_contexts = [
+            LazyKnownContext(FakeSequence(
+                self.evaluator, u'tuple',
+                (LazyTreeContext(self._defining_context, key_node),
+                 LazyTreeContext(self._defining_context, value_node))
+            )) for key_node, value_node in self.get_tree_entries()
+        ]
+
+        return ContextSet([FakeSequence(self.evaluator, u'list', lazy_contexts)])
+
+    def _dict_keys(self):
+        return ContextSet.from_sets(
+            self._defining_context.eval_node(k)
+            for k, v in self.get_tree_entries()
+        )
+
+    def get_mapping_item_contexts(self):
+        return self._dict_keys(), self._dict_values()
+
+
+class _FakeArray(SequenceLiteralContext):
+    def __init__(self, evaluator, container, type):
+        super(SequenceLiteralContext, self).__init__(evaluator)
+        self.array_type = type
+        self.atom = container
+        # TODO is this class really needed?
+
+
+class FakeSequence(_FakeArray):
+    def __init__(self, evaluator, array_type, lazy_context_list):
+        """
+        type should be one of "tuple", "list"
+        """
+        super(FakeSequence, self).__init__(evaluator, None, array_type)
+        self._lazy_context_list = lazy_context_list
+
+    def py__simple_getitem__(self, index):
+        if isinstance(index, slice):
+            return ContextSet([self])
+
+        with reraise_getitem_errors(IndexError, TypeError):
+            lazy_context = self._lazy_context_list[index]
+        return lazy_context.infer()
+
+    def py__iter__(self, contextualized_node=None):
+        return self._lazy_context_list
+
+    def py__bool__(self):
+        return bool(len(self._lazy_context_list))
+
+    def __repr__(self):
+        return "<%s of %s>" % (type(self).__name__, self._lazy_context_list)
+
+
+class FakeDict(_DictMixin, _FakeArray):
+    def __init__(self, evaluator, dct):
+        super(FakeDict, self).__init__(evaluator, dct, u'dict')
+        self._dct = dct
+
+    def py__iter__(self, contextualized_node=None):
+        for key in self._dct:
+            yield LazyKnownContext(compiled.create_simple_object(self.evaluator, key))
+
+    def py__simple_getitem__(self, index):
+        if is_py3 and self.evaluator.environment.version_info.major == 2:
+            # In Python 2 bytes and unicode compare.
+            if isinstance(index, bytes):
+                index_unicode = force_unicode(index)
+                try:
+                    return self._dct[index_unicode].infer()
+                except KeyError:
+                    pass
+            elif isinstance(index, str):
+                index_bytes = index.encode('utf-8')
+                try:
+                    return self._dct[index_bytes].infer()
+                except KeyError:
+                    pass
+
+        with reraise_getitem_errors(KeyError, TypeError):
+            lazy_context = self._dct[index]
+        return lazy_context.infer()
+
+    @publish_method('values')
+    def _values(self):
+        return ContextSet([FakeSequence(
+            self.evaluator, u'tuple',
+            [LazyKnownContexts(self._dict_values())]
+        )])
+
+    def _dict_values(self):
+        return ContextSet.from_sets(lazy_context.infer() for lazy_context in self._dct.values())
+
+    def _dict_keys(self):
+        return ContextSet.from_sets(lazy_context.infer() for lazy_context in self.py__iter__())
+
+    def get_mapping_item_contexts(self):
+        return self._dict_keys(), self._dict_values()
+
+    def exact_key_items(self):
+        return self._dct.items()
+
+
+class MergedArray(_FakeArray):
+    def __init__(self, evaluator, arrays):
+        super(MergedArray, self).__init__(evaluator, arrays, arrays[-1].array_type)
+        self._arrays = arrays
+
+    def py__iter__(self, contextualized_node=None):
+        for array in self._arrays:
+            for lazy_context in array.py__iter__():
+                yield lazy_context
+
+    def py__simple_getitem__(self, index):
+        return ContextSet.from_sets(lazy_context.infer() for lazy_context in self.py__iter__())
+
+    def get_tree_entries(self):
+        for array in self._arrays:
+            for a in array.get_tree_entries():
+                yield a
+
+    def __len__(self):
+        return sum(len(a) for a in self._arrays)
+
+
+def unpack_tuple_to_dict(context, types, exprlist):
+    """
+    Unpacking tuple assignments in for statements and expr_stmts.
+    """
+    if exprlist.type == 'name':
+        return {exprlist.value: types}
+    elif exprlist.type == 'atom' and exprlist.children[0] in ('(', '['):
+        return unpack_tuple_to_dict(context, types, exprlist.children[1])
+    elif exprlist.type in ('testlist', 'testlist_comp', 'exprlist',
+                           'testlist_star_expr'):
+        dct = {}
+        parts = iter(exprlist.children[::2])
+        n = 0
+        for lazy_context in types.iterate(exprlist):
+            n += 1
+            try:
+                part = next(parts)
+            except StopIteration:
+                # TODO this context is probably not right.
+                analysis.add(context, 'value-error-too-many-values', part,
+                             message="ValueError: too many values to unpack (expected %s)" % n)
+            else:
+                dct.update(unpack_tuple_to_dict(context, lazy_context.infer(), part))
+        has_parts = next(parts, None)
+        if types and has_parts is not None:
+            # TODO this context is probably not right.
+            analysis.add(context, 'value-error-too-few-values', has_parts,
+                         message="ValueError: need more than %s values to unpack" % n)
+        return dct
+    elif exprlist.type == 'power' or exprlist.type == 'atom_expr':
+        # Something like ``arr[x], var = ...``.
+        # This is something that is not yet supported, would also be difficult
+        # to write into a dict.
+        return {}
+    elif exprlist.type == 'star_expr':  # `a, *b, c = x` type unpackings
+        # Currently we're not supporting them.
+        return {}
+    raise NotImplementedError
+
+
+def check_array_additions(context, sequence):
+    """ Just a mapper function for the internal _check_array_additions """
+    if sequence.array_type not in ('list', 'set'):
+        # TODO also check for dict updates
+        return NO_CONTEXTS
+
+    return _check_array_additions(context, sequence)
+
+
+@evaluator_method_cache(default=NO_CONTEXTS)
+@debug.increase_indent
+def _check_array_additions(context, sequence):
+    """
+    Checks if a `Array` has "add" (append, insert, extend) statements:
+
+    >>> a = [""]
+    >>> a.append(1)
+    """
+    from jedi.evaluate import arguments
+
+    debug.dbg('Dynamic array search for %s' % sequence, color='MAGENTA')
+    module_context = context.get_root_context()
+    if not settings.dynamic_array_additions or isinstance(module_context, compiled.CompiledObject):
+        debug.dbg('Dynamic array search aborted.', color='MAGENTA')
+        return NO_CONTEXTS
+
+    def find_additions(context, arglist, add_name):
+        params = list(arguments.TreeArguments(context.evaluator, context, arglist).unpack())
+        result = set()
+        if add_name in ['insert']:
+            params = params[1:]
+        if add_name in ['append', 'add', 'insert']:
+            for key, lazy_context in params:
+                result.add(lazy_context)
+        elif add_name in ['extend', 'update']:
+            for key, lazy_context in params:
+                result |= set(lazy_context.infer().iterate())
+        return result
+
+    temp_param_add, settings.dynamic_params_for_other_modules = \
+        settings.dynamic_params_for_other_modules, False
+
+    is_list = sequence.name.string_name == 'list'
+    search_names = (['append', 'extend', 'insert'] if is_list else ['add', 'update'])
+
+    added_types = set()
+    for add_name in search_names:
+        try:
+            possible_names = module_context.tree_node.get_used_names()[add_name]
+        except KeyError:
+            continue
+        else:
+            for name in possible_names:
+                context_node = context.tree_node
+                if not (context_node.start_pos < name.start_pos < context_node.end_pos):
+                    continue
+                trailer = name.parent
+                power = trailer.parent
+                trailer_pos = power.children.index(trailer)
+                try:
+                    execution_trailer = power.children[trailer_pos + 1]
+                except IndexError:
+                    continue
+                else:
+                    if execution_trailer.type != 'trailer' \
+                            or execution_trailer.children[0] != '(' \
+                            or execution_trailer.children[1] == ')':
+                        continue
+
+                random_context = context.create_context(name)
+
+                with recursion.execution_allowed(context.evaluator, power) as allowed:
+                    if allowed:
+                        found = evaluate_call_of_leaf(
+                            random_context,
+                            name,
+                            cut_own_trailer=True
+                        )
+                        if sequence in found:
+                            # The arrays match. Now add the results
+                            added_types |= find_additions(
+                                random_context,
+                                execution_trailer.children[1],
+                                add_name
+                            )
+
+    # reset settings
+    settings.dynamic_params_for_other_modules = temp_param_add
+    debug.dbg('Dynamic array result %s' % added_types, color='MAGENTA')
+    return added_types
+
+
+def get_dynamic_array_instance(instance, arguments):
+    """Used for set() and list() instances."""
+    ai = _ArrayInstance(instance, arguments)
+    from jedi.evaluate import arguments
+    return arguments.ValuesArguments([ContextSet([ai])])
+
+
+class _ArrayInstance(HelperContextMixin):
+    """
+    Used for the usage of set() and list().
+    This is definitely a hack, but a good one :-)
+    It makes it possible to use set/list conversions.
+    """
+    def __init__(self, instance, var_args):
+        self.instance = instance
+        self.var_args = var_args
+
+    def py__class__(self):
+        tuple_, = self.instance.evaluator.builtins_module.py__getattribute__('tuple')
+        return tuple_
+
+    def py__iter__(self, contextualized_node=None):
+        var_args = self.var_args
+        try:
+            _, lazy_context = next(var_args.unpack())
+        except StopIteration:
+            pass
+        else:
+            for lazy in lazy_context.infer().iterate():
+                yield lazy
+
+        from jedi.evaluate import arguments
+        if isinstance(var_args, arguments.TreeArguments):
+            additions = _check_array_additions(var_args.context, self.instance)
+            for addition in additions:
+                yield addition
+
+    def iterate(self, contextualized_node=None, is_async=False):
+        return self.py__iter__(contextualized_node)
+
+
+class Slice(object):
+    def __init__(self, context, start, stop, step):
+        self._context = context
+        self._slice_object = None
+        # All of them are either a Precedence or None.
+        self._start = start
+        self._stop = stop
+        self._step = step
+
+    def __getattr__(self, name):
+        if self._slice_object is None:
+            context = compiled.builtin_from_name(self._context.evaluator, 'slice')
+            self._slice_object, = context.execute_evaluated()
+        return getattr(self._slice_object, name)
+
+    @property
+    def obj(self):
+        """
+        Imitate CompiledObject.obj behavior and return a ``builtin.slice()``
+        object.
+        """
+        def get(element):
+            if element is None:
+                return None
+
+            result = self._context.eval_node(element)
+            if len(result) != 1:
+                # For simplicity, we want slices to be clear defined with just
+                # one type.  Otherwise we will return an empty slice object.
+                raise IndexError
+
+            context, = result
+            return get_int_or_none(context)
+
+        try:
+            return slice(get(self._start), get(self._stop), get(self._step))
+        except IndexError:
+            return slice(None, None, None)

jedi/evaluate/context/klass.py

@@ -0,0 +1,344 @@
+"""
+Like described in the :mod:`parso.python.tree` module,
+there's a need for an ast like module to represent the states of parsed
+modules.
+
+But now there are also structures in Python that need a little bit more than
+that. An ``Instance`` for example is only a ``Class`` before it is
+instantiated. This class represents these cases.
+
+So, why is there also a ``Class`` class here? Well, there are decorators and
+they change classes in Python 3.
+
+Representation modules also define "magic methods". Those methods look like
+``py__foo__`` and are typically mappable to the Python equivalents ``__call__``
+and others. Here's a list:
+
+====================================== ========================================
+**Method**                             **Description**
+-------------------------------------- ----------------------------------------
+py__call__(arguments: Array)           On callable objects, returns types.
+py__bool__()                           Returns True/False/None; None means that
+                                       there's no certainty.
+py__bases__()                          Returns a list of base classes.
+py__iter__()                           Returns a generator of a set of types.
+py__class__()                          Returns the class of an instance.
+py__simple_getitem__(index: int/str)   Returns a a set of types of the index.
+                                       Can raise an IndexError/KeyError.
+py__getitem__(indexes: ContextSet)     Returns a a set of types of the index.
+py__file__()                           Only on modules. Returns None if does
+                                       not exist.
+py__package__() -> List[str]           Only on modules. For the import system.
+py__path__()                           Only on modules. For the import system.
+py__get__(call_object)                 Only on instances. Simulates
+                                       descriptors.
+py__doc__()                            Returns the docstring for a context.
+====================================== ========================================
+
+"""
+from jedi import debug
+from jedi._compatibility import use_metaclass
+from jedi.parser_utils import get_cached_parent_scope
+from jedi.evaluate.cache import evaluator_method_cache, CachedMetaClass, \
+    evaluator_method_generator_cache
+from jedi.evaluate import compiled
+from jedi.evaluate.lazy_context import LazyKnownContexts
+from jedi.evaluate.filters import ParserTreeFilter
+from jedi.evaluate.names import TreeNameDefinition, ContextName
+from jedi.evaluate.arguments import unpack_arglist, ValuesArguments
+from jedi.evaluate.base_context import ContextSet, iterator_to_context_set, \
+    NO_CONTEXTS
+from jedi.evaluate.context.function import FunctionAndClassBase
+from jedi.plugins import plugin_manager
+
+
+def apply_py__get__(context, instance, class_context):
+    try:
+        method = context.py__get__
+    except AttributeError:
+        yield context
+    else:
+        for descriptor_context in method(instance, class_context):
+            yield descriptor_context
+
+
+class ClassName(TreeNameDefinition):
+    def __init__(self, parent_context, tree_name, name_context, apply_decorators):
+        super(ClassName, self).__init__(parent_context, tree_name)
+        self._name_context = name_context
+        self._apply_decorators = apply_decorators
+
+    @iterator_to_context_set
+    def infer(self):
+        # We're using a different context to infer, so we cannot call super().
+        from jedi.evaluate.syntax_tree import tree_name_to_contexts
+        inferred = tree_name_to_contexts(
+            self.parent_context.evaluator, self._name_context, self.tree_name)
+
+        for result_context in inferred:
+            if self._apply_decorators:
+                for c in apply_py__get__(result_context,
+                                         instance=None,
+                                         class_context=self.parent_context):
+                    yield c
+            else:
+                yield result_context
+
+
+class ClassFilter(ParserTreeFilter):
+    name_class = ClassName
+
+    def __init__(self, *args, **kwargs):
+        self._is_instance = kwargs.pop('is_instance')  # Python 2 :/
+        super(ClassFilter, self).__init__(*args, **kwargs)
+
+    def _convert_names(self, names):
+        return [
+            self.name_class(
+                parent_context=self.context,
+                tree_name=name,
+                name_context=self._node_context,
+                apply_decorators=not self._is_instance,
+            ) for name in names
+        ]
+
+    def _equals_origin_scope(self):
+        node = self._origin_scope
+        while node is not None:
+            if node == self._parser_scope or node == self.context:
+                return True
+            node = get_cached_parent_scope(self._used_names, node)
+        return False
+
+    def _access_possible(self, name, from_instance=False):
+        # Filter for ClassVar variables
+        # TODO this is not properly done, yet. It just checks for the string
+        # ClassVar in the annotation, which can be quite imprecise. If we
+        # wanted to do this correct, we would have to resolve the ClassVar.
+        if not from_instance:
+            expr_stmt = name.get_definition()
+            if expr_stmt is not None and expr_stmt.type == 'expr_stmt':
+                annassign = expr_stmt.children[1]
+                if annassign.type == 'annassign':
+                    # TODO this is not proper matching
+                    if 'ClassVar' not in annassign.children[1].get_code():
+                        return False
+
+        # Filter for name mangling of private variables like __foo
+        return not name.value.startswith('__') or name.value.endswith('__') \
+            or self._equals_origin_scope()
+
+    def _filter(self, names, from_instance=False):
+        names = super(ClassFilter, self)._filter(names)
+        return [name for name in names if self._access_possible(name, from_instance)]
+
+
+class ClassMixin(object):
+    def is_class(self):
+        return True
+
+    def py__call__(self, arguments=None):
+        from jedi.evaluate.context import TreeInstance
+        if arguments is None:
+            arguments = ValuesArguments([])
+        return ContextSet([TreeInstance(self.evaluator, self.parent_context, self, arguments)])
+
+    def py__class__(self):
+        return compiled.builtin_from_name(self.evaluator, u'type')
+
+    @property
+    def name(self):
+        return ContextName(self, self.tree_node.name)
+
+    def py__name__(self):
+        return self.name.string_name
+
+    def get_param_names(self):
+        for context_ in self.py__getattribute__(u'__init__'):
+            if context_.is_function():
+                return list(context_.get_param_names())[1:]
+        return []
+
+    @evaluator_method_generator_cache()
+    def py__mro__(self):
+        mro = [self]
+        yield self
+        # TODO Do a proper mro resolution. Currently we are just listing
+        # classes. However, it's a complicated algorithm.
+        for lazy_cls in self.py__bases__():
+            # TODO there's multiple different mro paths possible if this yields
+            # multiple possibilities. Could be changed to be more correct.
+            for cls in lazy_cls.infer():
+                # TODO detect for TypeError: duplicate base class str,
+                # e.g.  `class X(str, str): pass`
+                try:
+                    mro_method = cls.py__mro__
+                except AttributeError:
+                    # TODO add a TypeError like:
+                    """
+                    >>> class Y(lambda: test): pass
+                    Traceback (most recent call last):
+                      File "<stdin>", line 1, in <module>
+                    TypeError: function() argument 1 must be code, not str
+                    >>> class Y(1): pass
+                    Traceback (most recent call last):
+                      File "<stdin>", line 1, in <module>
+                    TypeError: int() takes at most 2 arguments (3 given)
+                    """
+                    debug.warning('Super class of %s is not a class: %s', self, cls)
+                else:
+                    for cls_new in mro_method():
+                        if cls_new not in mro:
+                            mro.append(cls_new)
+                            yield cls_new
+
+    def get_filters(self, search_global=False, until_position=None,
+                    origin_scope=None, is_instance=False):
+        metaclasses = self.get_metaclasses()
+        if metaclasses:
+            for f in self.get_metaclass_filters(metaclasses):
+                yield f
+
+        if search_global:
+            yield self.get_global_filter(until_position, origin_scope)
+        else:
+            for cls in self.py__mro__():
+                if isinstance(cls, compiled.CompiledObject):
+                    for filter in cls.get_filters(is_instance=is_instance):
+                        yield filter
+                else:
+                    yield ClassFilter(
+                        self.evaluator, self, node_context=cls,
+                        origin_scope=origin_scope,
+                        is_instance=is_instance
+                    )
+        if not is_instance:
+            from jedi.evaluate.compiled import builtin_from_name
+            type_ = builtin_from_name(self.evaluator, u'type')
+            assert isinstance(type_, ClassContext)
+            if type_ != self:
+                for instance in type_.py__call__():
+                    instance_filters = instance.get_filters()
+                    # Filter out self filters
+                    next(instance_filters)
+                    next(instance_filters)
+                    yield next(instance_filters)
+
+    def get_signatures(self):
+        init_funcs = self.py__call__().py__getattribute__('__init__')
+        return [sig.bind(self) for sig in init_funcs.get_signatures()]
+
+    def get_global_filter(self, until_position=None, origin_scope=None):
+        return ParserTreeFilter(
+            self.evaluator,
+            context=self,
+            until_position=until_position,
+            origin_scope=origin_scope
+        )
+
+
+class ClassContext(use_metaclass(CachedMetaClass, ClassMixin, FunctionAndClassBase)):
+    """
+    This class is not only important to extend `tree.Class`, it is also a
+    important for descriptors (if the descriptor methods are evaluated or not).
+    """
+    api_type = u'class'
+
+    @evaluator_method_cache()
+    def list_type_vars(self):
+        found = []
+        arglist = self.tree_node.get_super_arglist()
+        if arglist is None:
+            return []
+
+        for stars, node in unpack_arglist(arglist):
+            if stars:
+                continue  # These are not relevant for this search.
+
+            from jedi.evaluate.gradual.annotation import find_unknown_type_vars
+            for type_var in find_unknown_type_vars(self.parent_context, node):
+                if type_var not in found:
+                    # The order matters and it's therefore a list.
+                    found.append(type_var)
+        return found
+
+    def _get_bases_arguments(self):
+        arglist = self.tree_node.get_super_arglist()
+        if arglist:
+            from jedi.evaluate import arguments
+            return arguments.TreeArguments(self.evaluator, self.parent_context, arglist)
+        return None
+
+    @evaluator_method_cache(default=())
+    def py__bases__(self):
+        args = self._get_bases_arguments()
+        if args is not None:
+            lst = [value for key, value in args.unpack() if key is None]
+            if lst:
+                return lst
+
+        if self.py__name__() == 'object' \
+                and self.parent_context == self.evaluator.builtins_module:
+            return []
+        return [LazyKnownContexts(
+            self.evaluator.builtins_module.py__getattribute__('object')
+        )]
+
+    def py__getitem__(self, index_context_set, contextualized_node):
+        from jedi.evaluate.gradual.typing import LazyGenericClass
+        if not index_context_set:
+            return ContextSet([self])
+        return ContextSet(
+            LazyGenericClass(
+                self,
+                index_context,
+                context_of_index=contextualized_node.context,
+            )
+            for index_context in index_context_set
+        )
+
+    def define_generics(self, type_var_dict):
+        from jedi.evaluate.gradual.typing import GenericClass
+
+        def remap_type_vars():
+            """
+            The TypeVars in the resulting classes have sometimes different names
+            and we need to check for that, e.g. a signature can be:
+
+            def iter(iterable: Iterable[_T]) -> Iterator[_T]: ...
+
+            However, the iterator is defined as Iterator[_T_co], which means it has
+            a different type var name.
+            """
+            for type_var in self.list_type_vars():
+                yield type_var_dict.get(type_var.py__name__(), NO_CONTEXTS)
+
+        if type_var_dict:
+            return ContextSet([GenericClass(
+                self,
+                generics=tuple(remap_type_vars())
+            )])
+        return ContextSet({self})
+
+    @plugin_manager.decorate()
+    def get_metaclass_filters(self, metaclass):
+        debug.dbg('Unprocessed metaclass %s', metaclass)
+        return []
+
+    @evaluator_method_cache(default=NO_CONTEXTS)
+    def get_metaclasses(self):
+        args = self._get_bases_arguments()
+        if args is not None:
+            m = [value for key, value in args.unpack() if key == 'metaclass']
+            metaclasses = ContextSet.from_sets(lazy_context.infer() for lazy_context in m)
+            metaclasses = ContextSet(m for m in metaclasses if m.is_class())
+            if metaclasses:
+                return metaclasses
+
+        for lazy_base in self.py__bases__():
+            for context in lazy_base.infer():
+                if context.is_class():
+                    contexts = context.get_metaclasses()
+                    if contexts:
+                        return contexts
+        return NO_CONTEXTS

jedi/evaluate/context/module.py

@@ -0,0 +1,283 @@
+import re
+import os
+
+from jedi import debug
+from jedi.evaluate.cache import evaluator_method_cache
+from jedi.evaluate.names import ContextNameMixin, AbstractNameDefinition
+from jedi.evaluate.filters import GlobalNameFilter, ParserTreeFilter, DictFilter, MergedFilter
+from jedi.evaluate import compiled
+from jedi.evaluate.base_context import TreeContext
+from jedi.evaluate.names import SubModuleName
+from jedi.evaluate.helpers import contexts_from_qualified_names
+from jedi.evaluate.compiled import create_simple_object
+from jedi.evaluate.base_context import ContextSet
+
+
+class _ModuleAttributeName(AbstractNameDefinition):
+    """
+    For module attributes like __file__, __str__ and so on.
+    """
+    api_type = u'instance'
+
+    def __init__(self, parent_module, string_name, string_value=None):
+        self.parent_context = parent_module
+        self.string_name = string_name
+        self._string_value = string_value
+
+    def infer(self):
+        if self._string_value is not None:
+            s = self._string_value
+            if self.parent_context.evaluator.environment.version_info.major == 2 \
+                    and not isinstance(s, bytes):
+                s = s.encode('utf-8')
+            return ContextSet([
+                create_simple_object(self.parent_context.evaluator, s)
+            ])
+        return compiled.get_string_context_set(self.parent_context.evaluator)
+
+
+class ModuleName(ContextNameMixin, AbstractNameDefinition):
+    start_pos = 1, 0
+
+    def __init__(self, context, name):
+        self._context = context
+        self._name = name
+
+    @property
+    def string_name(self):
+        return self._name
+
+
+def iter_module_names(evaluator, paths):
+    # Python modules/packages
+    for n in evaluator.compiled_subprocess.list_module_names(paths):
+        yield n
+
+    for path in paths:
+        try:
+            dirs = os.listdir(path)
+        except OSError:
+            # The file might not exist or reading it might lead to an error.
+            debug.warning("Not possible to list directory: %s", path)
+            continue
+        for name in dirs:
+            # Namespaces
+            if os.path.isdir(os.path.join(path, name)):
+                # pycache is obviously not an interestin namespace. Also the
+                # name must be a valid identifier.
+                # TODO use str.isidentifier, once Python 2 is removed
+                if name != '__pycache__' and not re.search(r'\W|^\d', name):
+                    yield name
+            # Stub files
+            if name.endswith('.pyi'):
+                if name != '__init__.pyi':
+                    yield name[:-4]
+
+
+class SubModuleDictMixin(object):
+    @evaluator_method_cache()
+    def sub_modules_dict(self):
+        """
+        Lists modules in the directory of this module (if this module is a
+        package).
+        """
+        names = {}
+        try:
+            method = self.py__path__
+        except AttributeError:
+            pass
+        else:
+            mods = iter_module_names(self.evaluator, method())
+            for name in mods:
+                # It's obviously a relative import to the current module.
+                names[name] = SubModuleName(self, name)
+
+        # In the case of an import like `from x.` we don't need to
+        # add all the variables, this is only about submodules.
+        return names
+
+
+class ModuleMixin(SubModuleDictMixin):
+    def get_filters(self, search_global=False, until_position=None, origin_scope=None):
+        yield MergedFilter(
+            ParserTreeFilter(
+                self.evaluator,
+                context=self,
+                until_position=until_position,
+                origin_scope=origin_scope
+            ),
+            GlobalNameFilter(self, self.tree_node),
+        )
+        yield DictFilter(self.sub_modules_dict())
+        yield DictFilter(self._module_attributes_dict())
+        for star_filter in self.iter_star_filters():
+            yield star_filter
+
+    def py__class__(self):
+        c, = contexts_from_qualified_names(self.evaluator, u'types', u'ModuleType')
+        return c
+
+    def is_module(self):
+        return True
+
+    def is_stub(self):
+        return False
+
+    @property
+    @evaluator_method_cache()
+    def name(self):
+        return ModuleName(self, self._string_name)
+
+    @property
+    def _string_name(self):
+        """ This is used for the goto functions. """
+        # TODO It's ugly that we even use this, the name is usually well known
+        # ahead so just pass it when create a ModuleContext.
+        if self._path is None:
+            return ''  # no path -> empty name
+        else:
+            sep = (re.escape(os.path.sep),) * 2
+            r = re.search(r'([^%s]*?)(%s__init__)?(\.pyi?|\.so)?$' % sep, self._path)
+            # Remove PEP 3149 names
+            return re.sub(r'\.[a-z]+-\d{2}[mud]{0,3}$', '', r.group(1))
+
+    @evaluator_method_cache()
+    def _module_attributes_dict(self):
+        names = ['__package__', '__doc__', '__name__']
+        # All the additional module attributes are strings.
+        dct = dict((n, _ModuleAttributeName(self, n)) for n in names)
+        file = self.py__file__()
+        if file is not None:
+            dct['__file__'] = _ModuleAttributeName(self, '__file__', file)
+        return dct
+
+    def iter_star_filters(self, search_global=False):
+        for star_module in self.star_imports():
+            yield next(star_module.get_filters(search_global))
+
+    # I'm not sure if the star import cache is really that effective anymore
+    # with all the other really fast import caches. Recheck. Also we would need
+    # to push the star imports into Evaluator.module_cache, if we reenable this.
+    @evaluator_method_cache([])
+    def star_imports(self):
+        from jedi.evaluate.imports import Importer
+
+        modules = []
+        for i in self.tree_node.iter_imports():
+            if i.is_star_import():
+                new = Importer(
+                    self.evaluator,
+                    import_path=i.get_paths()[-1],
+                    module_context=self,
+                    level=i.level
+                ).follow()
+
+                for module in new:
+                    if isinstance(module, ModuleContext):
+                        modules += module.star_imports()
+                modules += new
+        return modules
+
+    def get_qualified_names(self):
+        """
+        A module doesn't have a qualified name, but it's important to note that
+        it's reachable and not `None`. With this information we can add
+        qualified names on top for all context children.
+        """
+        return ()
+
+
+class ModuleContext(ModuleMixin, TreeContext):
+    api_type = u'module'
+    parent_context = None
+
+    def __init__(self, evaluator, module_node, file_io, string_names, code_lines, is_package=False):
+        super(ModuleContext, self).__init__(
+            evaluator,
+            parent_context=None,
+            tree_node=module_node
+        )
+        self.file_io = file_io
+        if file_io is None:
+            self._path = None
+        else:
+            self._path = file_io.path
+        self.string_names = string_names  # Optional[Tuple[str, ...]]
+        self.code_lines = code_lines
+        self.is_package = is_package
+
+    def is_stub(self):
+        if self._path is not None and self._path.endswith('.pyi'):
+            # Currently this is the way how we identify stubs when e.g. goto is
+            # used in them. This could be changed if stubs would be identified
+            # sooner and used as StubModuleContext.
+            return True
+        return super(ModuleContext, self).is_stub()
+
+    def py__name__(self):
+        if self.string_names is None:
+            return None
+        return '.'.join(self.string_names)
+
+    def py__file__(self):
+        """
+        In contrast to Python's __file__ can be None.
+        """
+        if self._path is None:
+            return None
+
+        return os.path.abspath(self._path)
+
+    def py__package__(self):
+        if self.is_package:
+            return self.string_names
+        return self.string_names[:-1]
+
+    def _py__path__(self):
+        # A namespace package is typically auto generated and ~10 lines long.
+        first_few_lines = ''.join(self.code_lines[:50])
+        # these are strings that need to be used for namespace packages,
+        # the first one is ``pkgutil``, the second ``pkg_resources``.
+        options = ('declare_namespace(__name__)', 'extend_path(__path__')
+        if options[0] in first_few_lines or options[1] in first_few_lines:
+            # It is a namespace, now try to find the rest of the
+            # modules on sys_path or whatever the search_path is.
+            paths = set()
+            for s in self.evaluator.get_sys_path():
+                other = os.path.join(s, self.name.string_name)
+                if os.path.isdir(other):
+                    paths.add(other)
+            if paths:
+                return list(paths)
+            # Nested namespace packages will not be supported. Nobody ever
+            # asked for it and in Python 3 they are there without using all the
+            # crap above.
+
+        # Default to the of this file.
+        file = self.py__file__()
+        assert file is not None  # Shouldn't be a package in the first place.
+        return [os.path.dirname(file)]
+
+    @property
+    def py__path__(self):
+        """
+        Not seen here, since it's a property. The callback actually uses a
+        variable, so use it like::
+
+            foo.py__path__(sys_path)
+
+        In case of a package, this returns Python's __path__ attribute, which
+        is a list of paths (strings).
+        Raises an AttributeError if the module is not a package.
+        """
+        if self.is_package:
+            return self._py__path__
+        else:
+            raise AttributeError('Only packages have __path__ attributes.')
+
+    def __repr__(self):
+        return "<%s: %s@%s-%s is_stub=%s>" % (
+            self.__class__.__name__, self._string_name,
+            self.tree_node.start_pos[0], self.tree_node.end_pos[0],
+            self.is_stub()
+        )

jedi/evaluate/context/namespace.py

@@ -0,0 +1,64 @@
+from jedi.evaluate.cache import evaluator_method_cache
+from jedi.evaluate.filters import DictFilter
+from jedi.evaluate.names import ContextNameMixin, AbstractNameDefinition
+from jedi.evaluate.base_context import Context
+from jedi.evaluate.context.module import SubModuleDictMixin
+
+
+class ImplicitNSName(ContextNameMixin, AbstractNameDefinition):
+    """
+    Accessing names for implicit namespace packages should infer to nothing.
+    This object will prevent Jedi from raising exceptions
+    """
+    def __init__(self, implicit_ns_context, string_name):
+        self._context = implicit_ns_context
+        self.string_name = string_name
+
+
+class ImplicitNamespaceContext(Context, SubModuleDictMixin):
+    """
+    Provides support for implicit namespace packages
+    """
+    # Is a module like every other module, because if you import an empty
+    # folder foobar it will be available as an object:
+    # <module 'foobar' (namespace)>.
+    api_type = u'module'
+    parent_context = None
+
+    def __init__(self, evaluator, fullname, paths):
+        super(ImplicitNamespaceContext, self).__init__(evaluator, parent_context=None)
+        self.evaluator = evaluator
+        self._fullname = fullname
+        self._paths = paths
+
+    def get_filters(self, search_global=False, until_position=None, origin_scope=None):
+        yield DictFilter(self.sub_modules_dict())
+
+    @property
+    @evaluator_method_cache()
+    def name(self):
+        string_name = self.py__package__()[-1]
+        return ImplicitNSName(self, string_name)
+
+    def py__file__(self):
+        return None
+
+    def py__package__(self):
+        """Return the fullname
+        """
+        return self._fullname.split('.')
+
+    def py__path__(self):
+        return self._paths
+
+    def py__name__(self):
+        return self._fullname
+
+    def is_namespace(self):
+        return True
+
+    def is_stub(self):
+        return False
+
+    def __repr__(self):
+        return '<%s: %s>' % (self.__class__.__name__, self._fullname)

jedi/evaluate/docstrings.py

@@ -1,11 +1,12 @@
 """
 Docstrings are another source of information for functions and classes.
 :mod:`jedi.evaluate.dynamic` tries to find all executions of functions, while
-the docstring parsing is much easier. There are two different types of
+the docstring parsing is much easier. There are three different types of
 docstrings that |jedi| understands:
 
 - `Sphinx <http://sphinx-doc.org/markup/desc.html#info-field-lists>`_
 - `Epydoc <http://epydoc.sourceforge.net/manual-fields.html>`_
+- `Numpydoc <https://github.com/numpy/numpy/blob/master/doc/HOWTO_DOCUMENT.rst.txt>`_
 
 For example, the sphinx annotation ``:type foo: str`` clearly states that the
 type of ``foo`` is ``str``.
@@ -14,23 +15,24 @@ As an addition to parameter searching, this module also provides return
 annotations.
 """
 
-from ast import literal_eval
 import re
+import warnings
 from textwrap import dedent
 
+from parso import parse, ParserSyntaxError
+
 from jedi._compatibility import u
-from jedi.common import unite
-from jedi.evaluate import context
-from jedi.evaluate.cache import memoize_default
-from jedi.parser import ParserWithRecovery, load_grammar
-from jedi.parser.tree import search_ancestor
-from jedi.common import indent_block
-from jedi.evaluate.iterable import SequenceLiteralContext, FakeSequence
+from jedi import debug
+from jedi.evaluate.utils import indent_block
+from jedi.evaluate.cache import evaluator_method_cache
+from jedi.evaluate.base_context import iterator_to_context_set, ContextSet, \
+    NO_CONTEXTS
+from jedi.evaluate.lazy_context import LazyKnownContexts
 
 
 DOCSTRING_PARAM_PATTERNS = [
     r'\s*:type\s+%s:\s*([^\n]+)',  # Sphinx
-    r'\s*:param\s+(\w+)\s+%s:[^\n]+',  # Sphinx param with type
+    r'\s*:param\s+(\w+)\s+%s:[^\n]*',  # Sphinx param with type
     r'\s*@type\s+%s:\s*([^\n]+)',  # Epydoc
 ]
 
@@ -42,27 +44,93 @@ DOCSTRING_RETURN_PATTERNS = [
 REST_ROLE_PATTERN = re.compile(r':[^`]+:`([^`]+)`')
 
 
-try:
-    from numpydoc.docscrape import NumpyDocString
-except ImportError:
-    def _search_param_in_numpydocstr(docstr, param_str):
-        return []
-else:
-    def _search_param_in_numpydocstr(docstr, param_str):
-        """Search `docstr` (in numpydoc format) for type(-s) of `param_str`."""
-        params = NumpyDocString(docstr)._parsed_data['Parameters']
-        for p_name, p_type, p_descr in params:
-            if p_name == param_str:
-                m = re.match('([^,]+(,[^,]+)*?)(,[ ]*optional)?$', p_type)
-                if m:
-                    p_type = m.group(1)
+_numpy_doc_string_cache = None
 
-                if p_type.startswith('{'):
-                    types = set(type(x).__name__ for x in literal_eval(p_type))
-                    return list(types)
-                else:
-                    return [p_type]
-        return []
+
+def _get_numpy_doc_string_cls():
+    global _numpy_doc_string_cache
+    if isinstance(_numpy_doc_string_cache, (ImportError, SyntaxError)):
+        raise _numpy_doc_string_cache
+    from numpydoc.docscrape import NumpyDocString
+    _numpy_doc_string_cache = NumpyDocString
+    return _numpy_doc_string_cache
+
+
+def _search_param_in_numpydocstr(docstr, param_str):
+    """Search `docstr` (in numpydoc format) for type(-s) of `param_str`."""
+    with warnings.catch_warnings():
+        warnings.simplefilter("ignore")
+        try:
+            # This is a non-public API. If it ever changes we should be
+            # prepared and return gracefully.
+            params = _get_numpy_doc_string_cls()(docstr)._parsed_data['Parameters']
+        except Exception:
+            return []
+    for p_name, p_type, p_descr in params:
+        if p_name == param_str:
+            m = re.match(r'([^,]+(,[^,]+)*?)(,[ ]*optional)?$', p_type)
+            if m:
+                p_type = m.group(1)
+            return list(_expand_typestr(p_type))
+    return []
+
+
+def _search_return_in_numpydocstr(docstr):
+    """
+    Search `docstr` (in numpydoc format) for type(-s) of function returns.
+    """
+    with warnings.catch_warnings():
+        warnings.simplefilter("ignore")
+        try:
+            doc = _get_numpy_doc_string_cls()(docstr)
+        except Exception:
+            return
+    try:
+        # This is a non-public API. If it ever changes we should be
+        # prepared and return gracefully.
+        returns = doc._parsed_data['Returns']
+        returns += doc._parsed_data['Yields']
+    except Exception:
+        return
+    for r_name, r_type, r_descr in returns:
+        # Return names are optional and if so the type is in the name
+        if not r_type:
+            r_type = r_name
+        for type_ in _expand_typestr(r_type):
+            yield type_
+
+
+def _expand_typestr(type_str):
+    """
+    Attempts to interpret the possible types in `type_str`
+    """
+    # Check if alternative types are specified with 'or'
+    if re.search(r'\bor\b', type_str):
+        for t in type_str.split('or'):
+            yield t.split('of')[0].strip()
+    # Check if like "list of `type`" and set type to list
+    elif re.search(r'\bof\b', type_str):
+        yield type_str.split('of')[0]
+    # Check if type has is a set of valid literal values eg: {'C', 'F', 'A'}
+    elif type_str.startswith('{'):
+        node = parse(type_str, version='3.7').children[0]
+        if node.type == 'atom':
+            for leaf in node.children[1].children:
+                if leaf.type == 'number':
+                    if '.' in leaf.value:
+                        yield 'float'
+                    else:
+                        yield 'int'
+                elif leaf.type == 'string':
+                    if 'b' in leaf.string_prefix.lower():
+                        yield 'bytes'
+                    else:
+                        yield 'str'
+                # Ignore everything else.
+
+    # Otherwise just work with what we have.
+    else:
+        yield type_str
 
 
 def _search_param_in_docstr(docstr, param_str):
@@ -90,8 +158,7 @@ def _search_param_in_docstr(docstr, param_str):
         if match:
             return [_strip_rst_role(match.group(1))]
 
-    return (_search_param_in_numpydocstr(docstr, param_str) or
-            [])
+    return _search_param_in_numpydocstr(docstr, param_str)
 
 
 def _strip_rst_role(type_str):
@@ -119,13 +186,17 @@ def _strip_rst_role(type_str):
 def _evaluate_for_statement_string(module_context, string):
     code = dedent(u("""
     def pseudo_docstring_stuff():
-        # Create a pseudo function for docstring statements.
-    {0}
+        '''
+        Create a pseudo function for docstring statements.
+        Need this docstring so that if the below part is not valid Python this
+        is still a function.
+        '''
+    {}
     """))
     if string is None:
         return []
 
-    for element in re.findall('((?:\w+\.)*\w+)\.', string):
+    for element in re.findall(r'((?:\w+\.)*\w+)\.', string):
         # Try to import module part in dotted name.
         # (e.g., 'threading' in 'threading.Thread').
         string = 'import %s\n' % element + string
@@ -133,25 +204,30 @@ def _evaluate_for_statement_string(module_context, string):
     # Take the default grammar here, if we load the Python 2.7 grammar here, it
     # will be impossible to use `...` (Ellipsis) as a token. Docstring types
     # don't need to conform with the current grammar.
-    p = ParserWithRecovery(load_grammar(), code.format(indent_block(string)))
+    debug.dbg('Parse docstring code %s', string, color='BLUE')
+    grammar = module_context.evaluator.latest_grammar
     try:
-        funcdef = p.module.subscopes[0]
+        module = grammar.parse(code.format(indent_block(string)), error_recovery=False)
+    except ParserSyntaxError:
+        return []
+    try:
+        funcdef = next(module.iter_funcdefs())
         # First pick suite, then simple_stmt and then the node,
         # which is also not the last item, because there's a newline.
         stmt = funcdef.children[-1].children[-1].children[-2]
     except (AttributeError, IndexError):
         return []
 
-    from jedi.evaluate.param import ValuesArguments
-    from jedi.evaluate.representation import FunctionContext
+    if stmt.type not in ('name', 'atom', 'atom_expr'):
+        return []
+
+    from jedi.evaluate.context import FunctionContext
     function_context = FunctionContext(
         module_context.evaluator,
         module_context,
         funcdef
     )
-    func_execution_context = function_context.get_function_execution(
-        ValuesArguments([])
-    )
+    func_execution_context = function_context.get_function_execution()
     # Use the module of the param.
     # TODO this module is not the module of the param in case of a function
     # call. In that case it's the module of the function call.
@@ -166,7 +242,10 @@ def _execute_types_in_stmt(module_context, stmt):
     contain is executed. (Used as type information).
     """
     definitions = module_context.eval_node(stmt)
-    return unite(_execute_array_values(module_context.evaluator, d) for d in definitions)
+    return ContextSet.from_sets(
+        _execute_array_values(module_context.evaluator, d)
+        for d in definitions
+    )
 
 
 def _execute_array_values(evaluator, array):
@@ -174,40 +253,59 @@ def _execute_array_values(evaluator, array):
     Tuples indicate that there's not just one return value, but the listed
     ones.  `(str, int)` means that it returns a tuple with both types.
     """
+    from jedi.evaluate.context.iterable import SequenceLiteralContext, FakeSequence
     if isinstance(array, SequenceLiteralContext):
         values = []
         for lazy_context in array.py__iter__():
-            objects = unite(_execute_array_values(evaluator, typ) for typ in lazy_context.infer())
-            values.append(context.LazyKnownContexts(objects))
-        return set([FakeSequence(evaluator, array.array_type, values)])
+            objects = ContextSet.from_sets(
+                _execute_array_values(evaluator, typ)
+                for typ in lazy_context.infer()
+            )
+            values.append(LazyKnownContexts(objects))
+        return {FakeSequence(evaluator, array.array_type, values)}
     else:
-        return array.execute_evaluated()
+        return array.execute_annotation()
 
 
-@memoize_default()
-def follow_param(module_context, param):
+@evaluator_method_cache()
+def infer_param(execution_context, param):
+    from jedi.evaluate.context.instance import InstanceArguments
+    from jedi.evaluate.context import FunctionExecutionContext
+
     def eval_docstring(docstring):
-        return set(
-            [p for param_str in _search_param_in_docstr(docstring, str(param.name))
-                for p in _evaluate_for_statement_string(module_context, param_str)]
+        return ContextSet(
+            p
+            for param_str in _search_param_in_docstr(docstring, param.name.value)
+            for p in _evaluate_for_statement_string(module_context, param_str)
         )
+    module_context = execution_context.get_root_context()
     func = param.get_parent_function()
-    types = eval_docstring(func.raw_doc)
-    if func.name.value == '__init__':
-        cls = search_ancestor(func, 'classdef')
-        if cls is not None:
-            types |= eval_docstring(cls.raw_doc)
+    if func.type == 'lambdef':
+        return NO_CONTEXTS
 
+    types = eval_docstring(execution_context.py__doc__())
+    if isinstance(execution_context, FunctionExecutionContext) \
+            and isinstance(execution_context.var_args, InstanceArguments) \
+            and execution_context.function_context.py__name__() == '__init__':
+        class_context = execution_context.var_args.instance.class_context
+        types |= eval_docstring(class_context.py__doc__())
+
+    debug.dbg('Found param types for docstring: %s', types, color='BLUE')
     return types
 
 
-@memoize_default()
-def find_return_types(module_context, func):
+@evaluator_method_cache()
+@iterator_to_context_set
+def infer_return_types(function_context):
     def search_return_in_docstr(code):
         for p in DOCSTRING_RETURN_PATTERNS:
             match = p.search(code)
             if match:
-                return _strip_rst_role(match.group(1))
+                yield _strip_rst_role(match.group(1))
+        # Check for numpy style return hint
+        for type_ in _search_return_in_numpydocstr(code):
+            yield type_
 
-    type_str = search_return_in_docstr(func.raw_doc)
-    return _evaluate_for_statement_string(module_context, type_str)
+    for type_str in search_return_in_docstr(function_context.py__doc__()):
+        for type_eval in _evaluate_for_statement_string(function_context.get_root_context(), type_str):
+            yield type_eval

jedi/evaluate/dynamic.py

@@ -14,45 +14,47 @@ It works as follows:
 
 - |Jedi| sees a param
 - search for function calls named ``foo``
-- execute these calls and check the input. This work with a ``ParamListener``.
+- execute these calls and check the input.
 """
 
-from jedi.parser import tree
 from jedi import settings
 from jedi import debug
-from jedi.evaluate.cache import memoize_default
+from jedi.evaluate.cache import evaluator_function_cache
 from jedi.evaluate import imports
-from jedi.evaluate.param import TreeArguments, create_default_param
-from jedi.common import to_list, unite
+from jedi.evaluate.arguments import TreeArguments
+from jedi.evaluate.param import create_default_params
+from jedi.evaluate.helpers import is_stdlib_path
+from jedi.evaluate.utils import to_list
+from jedi.parser_utils import get_parent_scope
+from jedi.evaluate.context import ModuleContext, instance
+from jedi.evaluate.base_context import ContextSet, NO_CONTEXTS
+from jedi.evaluate import recursion
 
 
 MAX_PARAM_SEARCHES = 20
 
 
-class ParamListener(object):
-    """
-    This listener is used to get the params for a function.
-    """
-    def __init__(self):
-        self.param_possibilities = []
-
-    def execute(self, params):
-        self.param_possibilities += params
-
-
-class MergedExecutedParams(object):
+class DynamicExecutedParams(object):
     """
     Simulates being a parameter while actually just being multiple params.
     """
-    def __init__(self, executed_params):
+
+    def __init__(self, evaluator, executed_params):
+        self.evaluator = evaluator
         self._executed_params = executed_params
 
     def infer(self):
-        return unite(p.infer() for p in self._executed_params)
+        with recursion.execution_allowed(self.evaluator, self) as allowed:
+            # We need to catch recursions that may occur, because an
+            # anonymous functions can create an anonymous parameter that is
+            # more or less self referencing.
+            if allowed:
+                return ContextSet.from_sets(p.infer() for p in self._executed_params)
+            return NO_CONTEXTS
 
 
 @debug.increase_indent
-def search_params(evaluator, parent_context, funcdef):
+def search_params(evaluator, execution_context, funcdef):
     """
     A dynamic search for param values. If you try to complete a type:
 
@@ -66,55 +68,71 @@ def search_params(evaluator, parent_context, funcdef):
     is.
     """
     if not settings.dynamic_params:
-        return set()
+        return create_default_params(execution_context, funcdef)
 
     evaluator.dynamic_params_depth += 1
     try:
-        debug.dbg('Dynamic param search in %s.', funcdef.name.value, color='MAGENTA')
-        module_context = parent_context.get_root_context()
-        function_executions = _search_function_executions(
-            evaluator,
-            module_context,
-            funcdef
-        )
-        if function_executions:
-            zipped_params = zip(*list(
-                function_execution.get_params()
-                for function_execution in function_executions
-            ))
-            params = [MergedExecutedParams(executed_params) for executed_params in zipped_params]
-            # Evaluate the ExecutedParams to types.
+        path = execution_context.get_root_context().py__file__()
+        if path is not None and is_stdlib_path(path):
+            # We don't want to search for usages in the stdlib. Usually people
+            # don't work with it (except if you are a core maintainer, sorry).
+            # This makes everything slower. Just disable it and run the tests,
+            # you will see the slowdown, especially in 3.6.
+            return create_default_params(execution_context, funcdef)
+
+        if funcdef.type == 'lambdef':
+            string_name = _get_lambda_name(funcdef)
+            if string_name is None:
+                return create_default_params(execution_context, funcdef)
         else:
-            params = [create_default_param(parent_context, p) for p in funcdef.params]
-        debug.dbg('Dynamic param result finished', color='MAGENTA')
+            string_name = funcdef.name.value
+        debug.dbg('Dynamic param search in %s.', string_name, color='MAGENTA')
+
+        try:
+            module_context = execution_context.get_root_context()
+            function_executions = _search_function_executions(
+                evaluator,
+                module_context,
+                funcdef,
+                string_name=string_name,
+            )
+            if function_executions:
+                zipped_params = zip(*list(
+                    function_execution.get_executed_params_and_issues()[0]
+                    for function_execution in function_executions
+                ))
+                params = [DynamicExecutedParams(evaluator, executed_params)
+                          for executed_params in zipped_params]
+                # Evaluate the ExecutedParams to types.
+            else:
+                return create_default_params(execution_context, funcdef)
+        finally:
+            debug.dbg('Dynamic param result finished', color='MAGENTA')
         return params
     finally:
         evaluator.dynamic_params_depth -= 1
 
 
-@memoize_default([], evaluator_is_first_arg=True)
+@evaluator_function_cache(default=None)
 @to_list
-def _search_function_executions(evaluator, module_context, funcdef):
+def _search_function_executions(evaluator, module_context, funcdef, string_name):
     """
     Returns a list of param names.
     """
-    from jedi.evaluate import representation as er
-
-    func_string_name = funcdef.name.value
     compare_node = funcdef
-    if func_string_name == '__init__':
-        cls = funcdef.get_parent_scope()
-        if isinstance(cls, tree.Class):
-            func_string_name = cls.name.value
+    if string_name == '__init__':
+        cls = get_parent_scope(funcdef)
+        if cls.type == 'classdef':
+            string_name = cls.name.value
             compare_node = cls
 
     found_executions = False
     i = 0
     for for_mod_context in imports.get_modules_containing_name(
-            evaluator, [module_context], func_string_name):
-        if not isinstance(module_context, er.ModuleContext):
+            evaluator, [module_context], string_name):
+        if not isinstance(module_context, ModuleContext):
             return
-        for name, trailer in _get_possible_nodes(for_mod_context, func_string_name):
+        for name, trailer in _get_possible_nodes(for_mod_context, string_name):
             i += 1
 
             # This is a simple way to stop Jedi's dynamic param recursion
@@ -135,9 +153,21 @@ def _search_function_executions(evaluator, module_context, funcdef):
             return
 
 
+def _get_lambda_name(node):
+    stmt = node.parent
+    if stmt.type == 'expr_stmt':
+        first_operator = next(stmt.yield_operators(), None)
+        if first_operator == '=':
+            first = stmt.children[0]
+            if first.type == 'name':
+                return first.value
+
+    return None
+
+
 def _get_possible_nodes(module_context, func_string_name):
     try:
-        names = module_context.tree_node.used_names[func_string_name]
+        names = module_context.tree_node.get_used_names()[func_string_name]
     except KeyError:
         return
 
@@ -149,16 +179,14 @@ def _get_possible_nodes(module_context, func_string_name):
 
 
 def _check_name_for_execution(evaluator, context, compare_node, name, trailer):
-    from jedi.evaluate import representation as er, instance
+    from jedi.evaluate.context.function import FunctionExecutionContext
 
     def create_func_excs():
         arglist = trailer.children[1]
         if arglist == ')':
-            arglist = ()
+            arglist = None
         args = TreeArguments(evaluator, context, arglist, trailer)
-        if value_node.type == 'funcdef':
-            yield value.get_function_execution(args)
-        else:
+        if value_node.type == 'classdef':
             created_instance = instance.TreeInstance(
                 evaluator,
                 value.parent_context,
@@ -167,18 +195,20 @@ def _check_name_for_execution(evaluator, context, compare_node, name, trailer):
             )
             for execution in created_instance.create_init_executions():
                 yield execution
+        else:
+            yield value.get_function_execution(args)
 
     for value in evaluator.goto_definitions(context, name):
         value_node = value.tree_node
         if compare_node == value_node:
             for func_execution in create_func_excs():
                 yield func_execution
-        elif isinstance(value.parent_context, er.FunctionExecutionContext) and \
+        elif isinstance(value.parent_context, FunctionExecutionContext) and \
                 compare_node.type == 'funcdef':
             # Here we're trying to find decorators by checking the first
             # parameter. It's not very generic though. Should find a better
             # solution that also applies to nested decorators.
-            params = value.parent_context.get_params()
+            params, _ = value.parent_context.get_executed_params_and_issues()
             if len(params) != 1:
                 continue
             values = params[0].infer()

jedi/evaluate/filters.py

@@ -3,121 +3,19 @@ Filters are objects that you can use to filter names in different scopes. They
 are needed for name resolution.
 """
 from abc import abstractmethod
+import weakref
 
-from jedi.parser.tree import search_ancestor
+from parso.tree import search_ancestor
+
+from jedi._compatibility import use_metaclass
 from jedi.evaluate import flow_analysis
-from jedi.common import to_list, unite
+from jedi.evaluate.base_context import ContextSet, Context, ContextWrapper, \
+    LazyContextWrapper
+from jedi.parser_utils import get_cached_parent_scope
+from jedi.evaluate.utils import to_list
+from jedi.evaluate.names import TreeNameDefinition, ParamName, AbstractNameDefinition
 
-
-class AbstractNameDefinition(object):
-    start_pos = None
-    string_name = None
-    parent_context = None
-    tree_name = None
-
-    @abstractmethod
-    def infer(self):
-        raise NotImplementedError
-
-    def get_root_context(self):
-        return self.parent_context.get_root_context()
-
-    def __repr__(self):
-        if self.start_pos is None:
-            return '<%s: %s>' % (self.__class__.__name__, self.string_name)
-        return '<%s: %s@%s>' % (self.__class__.__name__, self.string_name, self.start_pos)
-
-    def execute(self, arguments):
-        return unite(context.execute(arguments) for context in self.infer())
-
-    def execute_evaluated(self, *args, **kwargs):
-        return unite(context.execute_evaluated(*args, **kwargs) for context in self.infer())
-
-    @property
-    def api_type(self):
-        return self.parent_context.api_type
-
-
-class AbstractTreeName(AbstractNameDefinition):
-    def __init__(self, parent_context, tree_name):
-        self.parent_context = parent_context
-        self.tree_name = tree_name
-
-    @property
-    def string_name(self):
-        return self.tree_name.value
-
-    @property
-    def start_pos(self):
-        return self.tree_name.start_pos
-
-
-class ContextNameMixin(object):
-    def infer(self):
-        return set([self._context])
-
-    def get_root_context(self):
-        if self.parent_context is None:
-            return self._context
-        return super(ContextNameMixin, self).get_root_context()
-
-    @property
-    def api_type(self):
-        return self._context.api_type
-
-
-class ContextName(ContextNameMixin, AbstractTreeName):
-    def __init__(self, context, tree_name):
-        super(ContextName, self).__init__(context.parent_context, tree_name)
-        self._context = context
-
-
-class TreeNameDefinition(AbstractTreeName):
-    def get_parent_flow_context(self):
-        return self.parent_context
-
-    def infer(self):
-        # Refactor this, should probably be here.
-        from jedi.evaluate.finder import _name_to_types
-        return _name_to_types(self.parent_context.evaluator, self.parent_context, self.tree_name)
-
-    @property
-    def api_type(self):
-        definition = self.tree_name.get_definition()
-        return dict(
-            import_name='module',
-            import_from='module',
-            funcdef='function',
-            param='param',
-            classdef='class',
-        ).get(definition.type, 'statement')
-
-
-class ParamName(AbstractTreeName):
-    api_type = 'param'
-
-    def __init__(self, parent_context, tree_name):
-        self.parent_context = parent_context
-        self.tree_name = tree_name
-
-    def infer(self):
-        return self.get_param().infer()
-
-    def get_param(self):
-        params = self.parent_context.get_params()
-        param_node = search_ancestor(self.tree_name, 'param')
-        return params[param_node.position_nr]
-
-
-class AnonymousInstanceParamName(ParamName):
-    def infer(self):
-        param_node = search_ancestor(self.tree_name, 'param')
-        if param_node.position_nr == 0:
-            # This is a speed optimization, to return the self param (because
-            # it's known). This only affects anonymous instances.
-            return set([self.parent_context.instance])
-        else:
-            return self.get_param().infer()
+_definition_name_cache = weakref.WeakKeyDictionary()
 
 
 class AbstractFilter(object):
@@ -137,34 +35,70 @@ class AbstractFilter(object):
         raise NotImplementedError
 
 
+class FilterWrapper(object):
+    name_wrapper_class = None
+
+    def __init__(self, wrapped_filter):
+        self._wrapped_filter = wrapped_filter
+
+    def wrap_names(self, names):
+        return [self.name_wrapper_class(name) for name in names]
+
+    def get(self, name):
+        return self.wrap_names(self._wrapped_filter.get(name))
+
+    def values(self):
+        return self.wrap_names(self._wrapped_filter.values())
+
+
+def _get_definition_names(used_names, name_key):
+    try:
+        for_module = _definition_name_cache[used_names]
+    except KeyError:
+        for_module = _definition_name_cache[used_names] = {}
+
+    try:
+        return for_module[name_key]
+    except KeyError:
+        names = used_names.get(name_key, ())
+        result = for_module[name_key] = tuple(name for name in names if name.is_definition())
+        return result
+
+
 class AbstractUsedNamesFilter(AbstractFilter):
     name_class = TreeNameDefinition
 
     def __init__(self, context, parser_scope):
         self._parser_scope = parser_scope
-        self._used_names = self._parser_scope.get_root_node().used_names
+        self._module_node = self._parser_scope.get_root_node()
+        self._used_names = self._module_node.get_used_names()
         self.context = context
 
-    def get(self, name):
-        try:
-            names = self._used_names[str(name)]
-        except KeyError:
-            return []
-
-        return self._convert_names(self._filter(names))
+    def get(self, name, **filter_kwargs):
+        return self._convert_names(self._filter(
+            _get_definition_names(self._used_names, name),
+            **filter_kwargs
+        ))
 
     def _convert_names(self, names):
         return [self.name_class(self.context, name) for name in names]
 
-    def values(self):
-        return self._convert_names(name for name_list in self._used_names.values()
-                                   for name in self._filter(name_list))
+    def values(self, **filter_kwargs):
+        return self._convert_names(
+            name
+            for name_key in self._used_names
+            for name in self._filter(
+                _get_definition_names(self._used_names, name_key),
+                **filter_kwargs
+            )
+        )
 
     def __repr__(self):
         return '<%s: %s>' % (self.__class__.__name__, self.context)
 
 
 class ParserTreeFilter(AbstractUsedNamesFilter):
+    # TODO remove evaluator as an argument, it's not used.
     def __init__(self, evaluator, context, node_context=None, until_position=None,
                  origin_scope=None):
         """
@@ -186,18 +120,19 @@ class ParserTreeFilter(AbstractUsedNamesFilter):
         return list(self._check_flows(names))
 
     def _is_name_reachable(self, name):
-        if not name.is_definition():
-            return False
         parent = name.parent
         if parent.type == 'trailer':
             return False
         base_node = parent if parent.type in ('classdef', 'funcdef') else name
-        return base_node.get_parent_scope() == self._parser_scope
+        return get_cached_parent_scope(self._used_names, base_node) == self._parser_scope
 
     def _check_flows(self, names):
         for name in sorted(names, key=lambda name: name.start_pos, reverse=True):
             check = flow_analysis.reachability_check(
-                self._node_context, self._parser_scope, name, self._origin_scope
+                context=self._node_context,
+                context_scope=self._parser_scope,
+                node=name,
+                origin_scope=self._origin_scope
             )
             if check is not flow_analysis.UNREACHABLE:
                 yield name
@@ -229,20 +164,29 @@ class FunctionExecutionFilter(ParserTreeFilter):
                 yield TreeNameDefinition(self.context, name)
 
 
-class AnonymousInstanceFunctionExecutionFilter(FunctionExecutionFilter):
-    param_name = AnonymousInstanceParamName
-
-
 class GlobalNameFilter(AbstractUsedNamesFilter):
     def __init__(self, context, parser_scope):
         super(GlobalNameFilter, self).__init__(context, parser_scope)
 
+    def get(self, name):
+        try:
+            names = self._used_names[name]
+        except KeyError:
+            return []
+        return self._convert_names(self._filter(names))
+
     @to_list
     def _filter(self, names):
         for name in names:
             if name.parent.type == 'global_stmt':
                 yield name
 
+    def values(self):
+        return self._convert_names(
+            name for name_list in self._used_names.values()
+            for name in self._filter(name_list)
+        )
+
 
 class DictFilter(AbstractFilter):
     def __init__(self, dct):
@@ -250,18 +194,157 @@ class DictFilter(AbstractFilter):
 
     def get(self, name):
         try:
-            value = self._convert(name, self._dct[str(name)])
+            value = self._convert(name, self._dct[name])
         except KeyError:
             return []
-
-        return list(self._filter([value]))
+        else:
+            return list(self._filter([value]))
 
     def values(self):
-        return self._filter(self._convert(*item) for item in self._dct.items())
+        def yielder():
+            for item in self._dct.items():
+                try:
+                    yield self._convert(*item)
+                except KeyError:
+                    pass
+        return self._filter(yielder())
 
     def _convert(self, name, value):
         return value
 
+    def __repr__(self):
+        keys = ', '.join(self._dct.keys())
+        return '<%s: for {%s}>' % (self.__class__.__name__, keys)
+
+
+class MergedFilter(object):
+    def __init__(self, *filters):
+        self._filters = filters
+
+    def get(self, name):
+        return [n for filter in self._filters for n in filter.get(name)]
+
+    def values(self):
+        return [n for filter in self._filters for n in filter.values()]
+
+    def __repr__(self):
+        return '%s(%s)' % (self.__class__.__name__, ', '.join(str(f) for f in self._filters))
+
+
+class _BuiltinMappedMethod(Context):
+    """``Generator.__next__`` ``dict.values`` methods and so on."""
+    api_type = u'function'
+
+    def __init__(self, builtin_context, method, builtin_func):
+        super(_BuiltinMappedMethod, self).__init__(
+            builtin_context.evaluator,
+            parent_context=builtin_context
+        )
+        self._method = method
+        self._builtin_func = builtin_func
+
+    def py__call__(self, arguments):
+        # TODO add TypeError if params are given/or not correct.
+        return self._method(self.parent_context)
+
+    def __getattr__(self, name):
+        return getattr(self._builtin_func, name)
+
+
+class SpecialMethodFilter(DictFilter):
+    """
+    A filter for methods that are defined in this module on the corresponding
+    classes like Generator (for __next__, etc).
+    """
+    class SpecialMethodName(AbstractNameDefinition):
+        api_type = u'function'
+
+        def __init__(self, parent_context, string_name, value, builtin_context):
+            callable_, python_version = value
+            if python_version is not None and \
+                    python_version != parent_context.evaluator.environment.version_info.major:
+                raise KeyError
+
+            self.parent_context = parent_context
+            self.string_name = string_name
+            self._callable = callable_
+            self._builtin_context = builtin_context
+
+        def infer(self):
+            for filter in self._builtin_context.get_filters():
+                # We can take the first index, because on builtin methods there's
+                # always only going to be one name. The same is true for the
+                # inferred values.
+                for name in filter.get(self.string_name):
+                    builtin_func = next(iter(name.infer()))
+                    break
+                else:
+                    continue
+                break
+            return ContextSet([
+                _BuiltinMappedMethod(self.parent_context, self._callable, builtin_func)
+            ])
+
+    def __init__(self, context, dct, builtin_context):
+        super(SpecialMethodFilter, self).__init__(dct)
+        self.context = context
+        self._builtin_context = builtin_context
+        """
+        This context is what will be used to introspect the name, where as the
+        other context will be used to execute the function.
+
+        We distinguish, because we have to.
+        """
+
+    def _convert(self, name, value):
+        return self.SpecialMethodName(self.context, name, value, self._builtin_context)
+
+
+class _OverwriteMeta(type):
+    def __init__(cls, name, bases, dct):
+        super(_OverwriteMeta, cls).__init__(name, bases, dct)
+
+        base_dct = {}
+        for base_cls in reversed(cls.__bases__):
+            try:
+                base_dct.update(base_cls.overwritten_methods)
+            except AttributeError:
+                pass
+
+        for func in cls.__dict__.values():
+            try:
+                base_dct.update(func.registered_overwritten_methods)
+            except AttributeError:
+                pass
+        cls.overwritten_methods = base_dct
+
+
+class _AttributeOverwriteMixin(object):
+    def get_filters(self, search_global=False, *args, **kwargs):
+        yield SpecialMethodFilter(self, self.overwritten_methods, self._wrapped_context)
+
+        for filter in self._wrapped_context.get_filters(search_global):
+            yield filter
+
+
+class LazyAttributeOverwrite(use_metaclass(_OverwriteMeta, _AttributeOverwriteMixin,
+                                           LazyContextWrapper)):
+    def __init__(self, evaluator):
+        self.evaluator = evaluator
+
+
+class AttributeOverwrite(use_metaclass(_OverwriteMeta, _AttributeOverwriteMixin,
+                                       ContextWrapper)):
+    pass
+
+
+def publish_method(method_name, python_version_match=None):
+    def decorator(func):
+        dct = func.__dict__.setdefault('registered_overwritten_methods', {})
+        dct[method_name] = func, python_version_match
+        return func
+    return decorator
+
 
 def get_global_filters(evaluator, context, until_position, origin_scope):
     """
@@ -277,43 +360,41 @@ def get_global_filters(evaluator, context, until_position, origin_scope):
     ... def func():
     ...     y = None
     ... '''))
-    >>> module_node = script._get_module_node()
-    >>> scope = module_node.subscopes[0]
+    >>> module_node = script._module_node
+    >>> scope = next(module_node.iter_funcdefs())
     >>> scope
     <Function: func@3-5>
     >>> context = script._get_module().create_context(scope)
     >>> filters = list(get_global_filters(context.evaluator, context, (4, 0), None))
 
-    First we get the names names from the function scope.
+    First we get the names from the function scope.
 
-    >>> no_unicode_pprint(filters[0])
-    <ParserTreeFilter: <ModuleContext: <Module: None@2-5>>>
-    >>> sorted(str(n) for n in filters[0].values())
-    ['<TreeNameDefinition: func@(3, 4)>', '<TreeNameDefinition: x@(2, 0)>']
-    >>> filters[0]._until_position
+    >>> no_unicode_pprint(filters[0])  # doctest: +ELLIPSIS
+    MergedFilter(<ParserTreeFilter: ...>, <GlobalNameFilter: ...>)
+    >>> sorted(str(n) for n in filters[0].values())  # doctest: +NORMALIZE_WHITESPACE
+    ['<TreeNameDefinition: string_name=func start_pos=(3, 4)>',
+     '<TreeNameDefinition: string_name=x start_pos=(2, 0)>']
+    >>> filters[0]._filters[0]._until_position
     (4, 0)
+    >>> filters[0]._filters[1]._until_position
 
     Then it yields the names from one level "lower". In this example, this is
-    the module scope. As a side note, you can see, that the position in the
-    filter is now None, because typically the whole module is loaded before the
-    function is called.
+    the module scope (including globals).
+    As a side note, you can see, that the position in the filter is None on the
+    globals filter, because there the whole module is searched.
 
-    >>> filters[1].values()  # global names -> there are none in our example.
+    >>> list(filters[1].values())  # package modules -> Also empty.
     []
-    >>> list(filters[2].values())  # package modules -> Also empty.
-    []
-    >>> sorted(name.string_name for name in filters[3].values())  # Module attributes
-    ['__doc__', '__file__', '__name__', '__package__']
-    >>> print(filters[1]._until_position)
-    None
+    >>> sorted(name.string_name for name in filters[2].values())  # Module attributes
+    ['__doc__', '__name__', '__package__']
 
     Finally, it yields the builtin filter, if `include_builtin` is
     true (default).
 
-    >>> filters[4].values()                              #doctest: +ELLIPSIS
-    [<CompiledName: ...>, ...]
+    >>> list(filters[3].values())  # doctest: +ELLIPSIS
+    [...]
     """
-    from jedi.evaluate.representation import FunctionExecutionContext
+    from jedi.evaluate.context.function import FunctionExecutionContext
     while context is not None:
         # Names in methods cannot be resolved within the class.
         for filter in context.get_filters(
@@ -328,5 +409,4 @@ def get_global_filters(evaluator, context, until_position, origin_scope):
         context = context.parent_context
 
     # Add builtins to the global scope.
-    for filter in evaluator.BUILTINS.get_filters(search_global=True):
-        yield filter
+    yield next(evaluator.builtins_module.get_filters())

jedi/evaluate/finder.py

@@ -15,25 +15,26 @@ Unfortunately every other thing is being ignored (e.g. a == '' would be easy to
 check for -> a is a string). There's big potential in these checks.
 """
 
-from jedi.parser import tree
+from parso.python import tree
+from parso.tree import search_ancestor
 from jedi import debug
-from jedi.common import unite
 from jedi import settings
-from jedi.evaluate import representation as er
-from jedi.evaluate.instance import AbstractInstanceContext
 from jedi.evaluate import compiled
-from jedi.evaluate import pep0484
-from jedi.evaluate import iterable
-from jedi.evaluate import imports
 from jedi.evaluate import analysis
 from jedi.evaluate import flow_analysis
-from jedi.evaluate import param
+from jedi.evaluate.arguments import TreeArguments
 from jedi.evaluate import helpers
+from jedi.evaluate.context import iterable
 from jedi.evaluate.filters import get_global_filters
+from jedi.evaluate.names import TreeNameDefinition
+from jedi.evaluate.base_context import ContextSet, NO_CONTEXTS
+from jedi.parser_utils import is_scope, get_parent_scope
+from jedi.evaluate.gradual.conversion import convert_contexts
 
 
 class NameFinder(object):
-    def __init__(self, evaluator, context, name_context, name_or_str, position=None):
+    def __init__(self, evaluator, context, name_context, name_or_str,
+                 position=None, analysis_errors=True):
         self._evaluator = evaluator
         # Make sure that it's not just a syntax tree node.
         self._context = context
@@ -45,8 +46,8 @@ class NameFinder(object):
             self._string_name = name_or_str
         self._position = position
         self._found_predefined_types = None
+        self._analysis_errors = analysis_errors
 
-    @debug.increase_indent
     def find(self, filters, attribute_lookup):
         """
         :params bool attribute_lookup: Tell to logic if we're accessing the
@@ -55,14 +56,17 @@ class NameFinder(object):
         names = self.filter_name(filters)
         if self._found_predefined_types is not None and names:
             check = flow_analysis.reachability_check(
-                self._context, self._context.tree_node, self._name)
+                context=self._context,
+                context_scope=self._context.tree_node,
+                node=self._name,
+            )
             if check is flow_analysis.UNREACHABLE:
-                return set()
+                return NO_CONTEXTS
             return self._found_predefined_types
 
         types = self._names_to_types(names, attribute_lookup)
 
-        if not names and not types \
+        if not names and self._analysis_errors and not types \
                 and not (isinstance(self._name, tree.Name) and
                          isinstance(self._name.parent.parent, tree.Param)):
             if isinstance(self._name, tree.Name):
@@ -78,16 +82,50 @@ class NameFinder(object):
 
     def _get_origin_scope(self):
         if isinstance(self._name, tree.Name):
-            return self._name.get_parent_until(tree.Scope, reverse=True)
+            scope = self._name
+            while scope.parent is not None:
+                # TODO why if classes?
+                if not isinstance(scope, tree.Scope):
+                    break
+                scope = scope.parent
+            return scope
         else:
             return None
 
     def get_filters(self, search_global=False):
         origin_scope = self._get_origin_scope()
         if search_global:
-            return get_global_filters(self._evaluator, self._context, self._position, origin_scope)
+            position = self._position
+
+            # For functions and classes the defaults don't belong to the
+            # function and get evaluated in the context before the function. So
+            # make sure to exclude the function/class name.
+            if origin_scope is not None:
+                ancestor = search_ancestor(origin_scope, 'funcdef', 'classdef', 'lambdef')
+                lambdef = None
+                if ancestor == 'lambdef':
+                    # For lambdas it's even more complicated since parts will
+                    # be evaluated later.
+                    lambdef = ancestor
+                    ancestor = search_ancestor(origin_scope, 'funcdef', 'classdef')
+                if ancestor is not None:
+                    colon = ancestor.children[-2]
+                    if position is not None and position < colon.start_pos:
+                        if lambdef is None or position < lambdef.children[-2].start_pos:
+                            position = ancestor.start_pos
+
+            return get_global_filters(self._evaluator, self._context, position, origin_scope)
         else:
-            return self._context.get_filters(search_global, self._position, origin_scope=origin_scope)
+            return self._get_context_filters(origin_scope)
+
+    def _get_context_filters(self, origin_scope):
+        for f in self._context.get_filters(False, self._position, origin_scope=origin_scope):
+            yield f
+        # This covers the case where a stub files are incomplete.
+        if self._context.is_stub():
+            for c in convert_contexts(ContextSet({self._context})):
+                for f in c.get_filters():
+                    yield f
 
     def filter_name(self, filters):
         """
@@ -95,12 +133,13 @@ class NameFinder(object):
         ``filters``), until a name fits.
         """
         names = []
-        if self._context.predefined_names:
-            # TODO is this ok? node might not always be a tree.Name
+        # This paragraph is currently needed for proper branch evaluation
+        # (static analysis).
+        if self._context.predefined_names and isinstance(self._name, tree.Name):
             node = self._name
-            while node is not None and not node.is_scope():
+            while node is not None and not is_scope(node):
                 node = node.parent
-                if node.type in ("if_stmt", "for_stmt", "comp_for"):
+                if node.type in ("if_stmt", "for_stmt", "comp_for", 'sync_comp_for'):
                     try:
                         name_dict = self._context.predefined_names[node]
                         types = name_dict[self._string_name]
@@ -111,17 +150,30 @@ class NameFinder(object):
                         break
 
         for filter in filters:
-            names = filter.get(self._name)
+            names = filter.get(self._string_name)
             if names:
+                if len(names) == 1:
+                    n, = names
+                    if isinstance(n, TreeNameDefinition):
+                        # Something somewhere went terribly wrong. This
+                        # typically happens when using goto on an import in an
+                        # __init__ file. I think we need a better solution, but
+                        # it's kind of hard, because for Jedi it's not clear
+                        # that that name has not been defined, yet.
+                        if n.tree_name == self._name:
+                            def_ = self._name.get_definition()
+                            if def_ is not None and def_.type == 'import_from':
+                                continue
                 break
-        debug.dbg('finder.filter_name "%s" in (%s): %s@%s', self._string_name,
-                  self._context, names, self._position)
+
+        debug.dbg('finder.filter_name %s in (%s): %s@%s',
+                  self._string_name, self._context, names, self._position)
         return list(names)
 
     def _check_getattr(self, inst):
         """Checks for both __getattr__ and __getattribute__ methods"""
         # str is important, because it shouldn't be `Name`!
-        name = compiled.create(self._evaluator, self._string_name)
+        name = compiled.create_simple_object(self._evaluator, self._string_name)
 
         # This is a little bit special. `__getattribute__` is in Python
         # executed before `__getattr__`. But: I know no use case, where
@@ -130,150 +182,35 @@ class NameFinder(object):
         # We are inversing this, because a hand-crafted `__getattribute__`
         # could still call another hand-crafted `__getattr__`, but not the
         # other way around.
-        names = (inst.get_function_slot_names('__getattr__') or
-                 inst.get_function_slot_names('__getattribute__'))
+        names = (inst.get_function_slot_names(u'__getattr__') or
+                 inst.get_function_slot_names(u'__getattribute__'))
         return inst.execute_function_slots(names, name)
 
     def _names_to_types(self, names, attribute_lookup):
-        types = set()
+        contexts = ContextSet.from_sets(name.infer() for name in names)
 
-        types = unite(name.infer() for name in names)
-
-        debug.dbg('finder._names_to_types: %s -> %s', names, types)
-        if not names and isinstance(self._context, AbstractInstanceContext):
+        debug.dbg('finder._names_to_types: %s -> %s', names, contexts)
+        if not names and self._context.is_instance() and not self._context.is_compiled():
             # handling __getattr__ / __getattribute__
             return self._check_getattr(self._context)
 
         # Add isinstance and other if/assert knowledge.
-        if not types and isinstance(self._name, tree.Name) and \
-                not isinstance(self._name_context, AbstractInstanceContext):
+        if not contexts and isinstance(self._name, tree.Name) and \
+                not self._name_context.is_instance() and not self._context.is_compiled():
             flow_scope = self._name
-            base_node = self._name_context.tree_node
-            if base_node.type == 'comp_for':
-                return types
+            base_nodes = [self._name_context.tree_node]
+
+            if any(b.type in ('comp_for', 'sync_comp_for') for b in base_nodes):
+                return contexts
             while True:
-                flow_scope = flow_scope.get_parent_scope(include_flows=True)
+                flow_scope = get_parent_scope(flow_scope, include_flows=True)
                 n = _check_flow_information(self._name_context, flow_scope,
                                             self._name, self._position)
                 if n is not None:
                     return n
-                if flow_scope == base_node:
+                if flow_scope in base_nodes:
                     break
-        return types
-
-
-def _name_to_types(evaluator, context, tree_name):
-    types = []
-    node = tree_name.get_definition()
-    if node.isinstance(tree.ForStmt):
-        types = pep0484.find_type_from_comment_hint_for(context, node, tree_name)
-        if types:
-            return types
-    if node.isinstance(tree.WithStmt):
-        types = pep0484.find_type_from_comment_hint_with(context, node, tree_name)
-        if types:
-            return types
-    if node.type in ('for_stmt', 'comp_for'):
-        try:
-            types = context.predefined_names[node][tree_name.value]
-        except KeyError:
-            container_types = context.eval_node(node.children[3])
-            for_types = iterable.py__iter__types(evaluator, container_types, node.children[3])
-            types = check_tuple_assignments(evaluator, for_types, tree_name)
-    elif node.isinstance(tree.ExprStmt):
-        types = _remove_statements(evaluator, context, node, tree_name)
-    elif node.isinstance(tree.WithStmt):
-        types = context.eval_node(node.node_from_name(tree_name))
-    elif isinstance(node, tree.Import):
-        types = imports.infer_import(context, tree_name)
-    elif node.type in ('funcdef', 'classdef'):
-        types = _apply_decorators(evaluator, context, node)
-    elif node.type == 'global_stmt':
-        context = evaluator.create_context(context, tree_name)
-        finder = NameFinder(evaluator, context, context, str(tree_name))
-        filters = finder.get_filters(search_global=True)
-        # For global_stmt lookups, we only need the first possible scope,
-        # which means the function itself.
-        filters = [next(filters)]
-        types += finder.find(filters, attribute_lookup=False)
-    elif isinstance(node, tree.TryStmt):
-        # TODO an exception can also be a tuple. Check for those.
-        # TODO check for types that are not classes and add it to
-        # the static analysis report.
-        exceptions = context.eval_node(tree_name.get_previous_sibling().get_previous_sibling())
-        types = unite(
-            evaluator.execute(t, param.ValuesArguments([]))
-            for t in exceptions
-        )
-    else:
-        raise ValueError("Should not happen.")
-    return types
-
-
-def _apply_decorators(evaluator, context, node):
-    """
-    Returns the function, that should to be executed in the end.
-    This is also the places where the decorators are processed.
-    """
-    if node.type == 'classdef':
-        decoratee_context = er.ClassContext(
-            evaluator,
-            parent_context=context,
-            classdef=node
-        )
-    else:
-        decoratee_context = er.FunctionContext(
-            evaluator,
-            parent_context=context,
-            funcdef=node
-        )
-    initial = values = set([decoratee_context])
-    for dec in reversed(node.get_decorators()):
-        debug.dbg('decorator: %s %s', dec, values)
-        dec_values = context.eval_node(dec.children[1])
-        trailer_nodes = dec.children[2:-1]
-        if trailer_nodes:
-            # Create a trailer and evaluate it.
-            trailer = tree.Node('trailer', trailer_nodes)
-            trailer.parent = dec
-            dec_values = evaluator.eval_trailer(context, dec_values, trailer)
-
-        if not len(dec_values):
-            debug.warning('decorator not found: %s on %s', dec, node)
-            return initial
-
-        values = unite(dec_value.execute(param.ValuesArguments([values]))
-                       for dec_value in dec_values)
-        if not len(values):
-            debug.warning('not possible to resolve wrappers found %s', node)
-            return initial
-
-        debug.dbg('decorator end %s', values)
-    return values
-
-
-def _remove_statements(evaluator, context, stmt, name):
-    """
-    This is the part where statements are being stripped.
-
-    Due to lazy evaluation, statements like a = func; b = a; b() have to be
-    evaluated.
-    """
-    types = set()
-    check_instance = None
-
-    pep0484types = \
-        pep0484.find_type_from_comment_hint_assign(context, stmt, name)
-    if pep0484types:
-        return pep0484types
-    types |= context.eval_stmt(stmt, seek_name=name)
-
-    if check_instance is not None:
-        # class renames
-        types = set([er.get_instance_el(evaluator, check_instance, a, True)
-                     if isinstance(a, (er.Function, tree.Function))
-                     else a for a in types])
-    return types
+        return contexts
 
 
 def _check_flow_information(context, flow, search_name, pos):
@@ -289,11 +226,11 @@ def _check_flow_information(context, flow, search_name, pos):
         return None
 
     result = None
-    if flow.is_scope():
+    if is_scope(flow):
         # Check for asserts.
         module_node = flow.get_root_node()
         try:
-            names = module_node.used_names[search_name.value]
+            names = module_node.get_used_names()[search_name.value]
         except KeyError:
             return None
         names = reversed([
@@ -302,13 +239,13 @@ def _check_flow_information(context, flow, search_name, pos):
         ])
 
         for name in names:
-            ass = tree.search_ancestor(name, 'assert_stmt')
+            ass = search_ancestor(name, 'assert_stmt')
             if ass is not None:
-                result = _check_isinstance_type(context, ass.assertion(), search_name)
+                result = _check_isinstance_type(context, ass.assertion, search_name)
                 if result is not None:
                     return result
 
-    if isinstance(flow, (tree.IfStmt, tree.WhileStmt)):
+    if flow.type in ('if_stmt', 'while_stmt'):
         potential_ifs = [c for c in flow.children[1::4] if c != ':']
         for if_test in reversed(potential_ifs):
             if search_name.start_pos > if_test.end_pos:
@@ -322,13 +259,13 @@ def _check_isinstance_type(context, element, search_name):
         # this might be removed if we analyze and, etc
         assert len(element.children) == 2
         first, trailer = element.children
-        assert isinstance(first, tree.Name) and first.value == 'isinstance'
+        assert first.type == 'name' and first.value == 'isinstance'
         assert trailer.type == 'trailer' and trailer.children[0] == '('
         assert len(trailer.children) == 3
 
         # arglist stuff
         arglist = trailer.children[1]
-        args = param.TreeArguments(context.evaluator, context, arglist, trailer)
+        args = TreeArguments(context.evaluator, context, arglist, trailer)
         param_list = list(args.unpack())
         # Disallow keyword arguments
         assert len(param_list) == 2
@@ -338,37 +275,16 @@ def _check_isinstance_type(context, element, search_name):
         is_instance_call = helpers.call_of_leaf(lazy_context_object.data)
         # Do a simple get_code comparison. They should just have the same code,
         # and everything will be all right.
-        assert is_instance_call.get_code(normalized=True) == call.get_code(normalized=True)
+        normalize = context.evaluator.grammar._normalize
+        assert normalize(is_instance_call) == normalize(call)
     except AssertionError:
         return None
 
-    result = set()
+    context_set = NO_CONTEXTS
     for cls_or_tup in lazy_context_cls.infer():
-        if isinstance(cls_or_tup, iterable.AbstractSequence) and \
-                cls_or_tup.array_type == 'tuple':
+        if isinstance(cls_or_tup, iterable.Sequence) and cls_or_tup.array_type == 'tuple':
             for lazy_context in cls_or_tup.py__iter__():
-                for context in lazy_context.infer():
-                    result |= context.execute_evaluated()
+                context_set |= lazy_context.infer().execute_evaluated()
         else:
-            result |= cls_or_tup.execute_evaluated()
-    return result
-
-
-def check_tuple_assignments(evaluator, types, name):
-    """
-    Checks if tuples are assigned.
-    """
-    lazy_context = None
-    for index, node in name.assignment_indexes():
-        iterated = iterable.py__iter__(evaluator, types, node)
-        for _ in range(index + 1):
-            try:
-                lazy_context = next(iterated)
-            except StopIteration:
-                # We could do this with the default param in next. But this
-                # would allow this loop to run for a very long time if the
-                # index number is high. Therefore break if the loop is
-                # finished.
-                return set()
-        types = lazy_context.infer()
-    return types
+            context_set |= cls_or_tup.execute_evaluated()
+    return context_set

jedi/evaluate/flow_analysis.py

@@ -1,3 +1,7 @@
+from jedi.parser_utils import get_flow_branch_keyword, is_scope, get_parent_scope
+from jedi.evaluate.recursion import execution_allowed
+
+
 class Status(object):
     lookup_table = {}
 
@@ -31,14 +35,14 @@ UNSURE = Status(None, 'unsure')
 
 def _get_flow_scopes(node):
     while True:
-        node = node.get_parent_scope(include_flows=True)
-        if node is None or node.is_scope():
+        node = get_parent_scope(node, include_flows=True)
+        if node is None or is_scope(node):
             return
         yield node
 
 
 def reachability_check(context, context_scope, node, origin_scope=None):
-    first_flow_scope = node.get_parent_scope(include_flows=True)
+    first_flow_scope = get_parent_scope(node, include_flows=True)
     if origin_scope is not None:
         origin_flow_scopes = list(_get_flow_scopes(origin_scope))
         node_flow_scopes = list(_get_flow_scopes(node))
@@ -46,8 +50,8 @@ def reachability_check(context, context_scope, node, origin_scope=None):
         branch_matches = True
         for flow_scope in origin_flow_scopes:
             if flow_scope in node_flow_scopes:
-                node_keyword = flow_scope.get_branch_keyword(node)
-                origin_keyword = flow_scope.get_branch_keyword(origin_scope)
+                node_keyword = get_flow_branch_keyword(flow_scope, node)
+                origin_keyword = get_flow_branch_keyword(flow_scope, origin_scope)
                 branch_matches = node_keyword == origin_keyword
                 if flow_scope.type == 'if_stmt':
                     if not branch_matches:
@@ -56,7 +60,8 @@ def reachability_check(context, context_scope, node, origin_scope=None):
                     if not branch_matches and origin_keyword == 'else' \
                             and node_keyword == 'except':
                         return UNREACHABLE
-                break
+                if branch_matches:
+                    break
 
         # Direct parents get resolved, we filter scopes that are separate
         # branches.  This makes sense for autocompletion and static analysis.
@@ -76,14 +81,14 @@ def reachability_check(context, context_scope, node, origin_scope=None):
 def _break_check(context, context_scope, flow_scope, node):
     reachable = REACHABLE
     if flow_scope.type == 'if_stmt':
-        if flow_scope.node_after_else(node):
-            for check_node in flow_scope.check_nodes():
+        if flow_scope.is_node_after_else(node):
+            for check_node in flow_scope.get_test_nodes():
                 reachable = _check_if(context, check_node)
                 if reachable in (REACHABLE, UNSURE):
                     break
             reachable = reachable.invert()
         else:
-            flow_node = flow_scope.node_in_which_check_node(node)
+            flow_node = flow_scope.get_corresponding_test_node(node)
             if flow_node is not None:
                 reachable = _check_if(context, flow_node)
     elif flow_scope.type in ('try_stmt', 'while_stmt'):
@@ -94,16 +99,20 @@ def _break_check(context, context_scope, flow_scope, node):
         return reachable
 
     if context_scope != flow_scope and context_scope != flow_scope.parent:
-        flow_scope = flow_scope.get_parent_scope(include_flows=True)
+        flow_scope = get_parent_scope(flow_scope, include_flows=True)
         return reachable & _break_check(context, context_scope, flow_scope, node)
     else:
         return reachable
 
 
 def _check_if(context, node):
-    types = context.eval_node(node)
-    values = set(x.py__bool__() for x in types)
-    if len(values) == 1:
-        return Status.lookup_table[values.pop()]
-    else:
-        return UNSURE
+    with execution_allowed(context.evaluator, node) as allowed:
+        if not allowed:
+            return UNSURE
+
+        types = context.eval_node(node)
+        values = set(x.py__bool__() for x in types)
+        if len(values) == 1:
+            return Status.lookup_table[values.pop()]
+        else:
+            return UNSURE

jedi/evaluate/gradual/annotation.py

@@ -0,0 +1,405 @@
+"""
+PEP 0484 ( https://www.python.org/dev/peps/pep-0484/ ) describes type hints
+through function annotations. There is a strong suggestion in this document
+that only the type of type hinting defined in PEP0484 should be allowed
+as annotations in future python versions.
+"""
+
+import re
+
+from parso import ParserSyntaxError, parse
+
+from jedi._compatibility import force_unicode
+from jedi.evaluate.cache import evaluator_method_cache
+from jedi.evaluate.base_context import ContextSet, NO_CONTEXTS
+from jedi.evaluate.gradual.typing import TypeVar, LazyGenericClass, \
+    AbstractAnnotatedClass
+from jedi.evaluate.gradual.typing import GenericClass
+from jedi.evaluate.helpers import is_string
+from jedi.evaluate.compiled import builtin_from_name
+from jedi import debug
+from jedi import parser_utils
+
+
+def eval_annotation(context, annotation):
+    """
+    Evaluates an annotation node. This means that it evaluates the part of
+    `int` here:
+
+        foo: int = 3
+
+    Also checks for forward references (strings)
+    """
+    context_set = context.eval_node(annotation)
+    if len(context_set) != 1:
+        debug.warning("Eval'ed typing index %s should lead to 1 object, "
+                      " not %s" % (annotation, context_set))
+        return context_set
+
+    evaled_context = list(context_set)[0]
+    if is_string(evaled_context):
+        result = _get_forward_reference_node(context, evaled_context.get_safe_value())
+        if result is not None:
+            return context.eval_node(result)
+    return context_set
+
+
+def _evaluate_annotation_string(context, string, index=None):
+    node = _get_forward_reference_node(context, string)
+    if node is None:
+        return NO_CONTEXTS
+
+    context_set = context.eval_node(node)
+    if index is not None:
+        context_set = context_set.filter(
+            lambda context: context.array_type == u'tuple'  # noqa
+                            and len(list(context.py__iter__())) >= index
+        ).py__simple_getitem__(index)
+    return context_set
+
+
+def _get_forward_reference_node(context, string):
+    try:
+        new_node = context.evaluator.grammar.parse(
+            force_unicode(string),
+            start_symbol='eval_input',
+            error_recovery=False
+        )
+    except ParserSyntaxError:
+        debug.warning('Annotation not parsed: %s' % string)
+        return None
+    else:
+        module = context.tree_node.get_root_node()
+        parser_utils.move(new_node, module.end_pos[0])
+        new_node.parent = context.tree_node
+        return new_node
+
+
+def _split_comment_param_declaration(decl_text):
+    """
+    Split decl_text on commas, but group generic expressions
+    together.
+
+    For example, given "foo, Bar[baz, biz]" we return
+    ['foo', 'Bar[baz, biz]'].
+
+    """
+    try:
+        node = parse(decl_text, error_recovery=False).children[0]
+    except ParserSyntaxError:
+        debug.warning('Comment annotation is not valid Python: %s' % decl_text)
+        return []
+
+    if node.type == 'name':
+        return [node.get_code().strip()]
+
+    params = []
+    try:
+        children = node.children
+    except AttributeError:
+        return []
+    else:
+        for child in children:
+            if child.type in ['name', 'atom_expr', 'power']:
+                params.append(child.get_code().strip())
+
+    return params
+
+
+@evaluator_method_cache()
+def infer_param(execution_context, param):
+    contexts = _infer_param(execution_context, param)
+    evaluator = execution_context.evaluator
+    if param.star_count == 1:
+        tuple_ = builtin_from_name(evaluator, 'tuple')
+        return ContextSet([GenericClass(
+            tuple_,
+            generics=(contexts,),
+        ) for c in contexts])
+    elif param.star_count == 2:
+        dct = builtin_from_name(evaluator, 'dict')
+        return ContextSet([GenericClass(
+            dct,
+            generics=(ContextSet([builtin_from_name(evaluator, 'str')]), contexts),
+        ) for c in contexts])
+        pass
+    return contexts
+
+
+def _infer_param(execution_context, param):
+    """
+    Infers the type of a function parameter, using type annotations.
+    """
+    annotation = param.annotation
+    if annotation is None:
+        # If no Python 3-style annotation, look for a Python 2-style comment
+        # annotation.
+        # Identify parameters to function in the same sequence as they would
+        # appear in a type comment.
+        all_params = [child for child in param.parent.children
+                      if child.type == 'param']
+
+        node = param.parent.parent
+        comment = parser_utils.get_following_comment_same_line(node)
+        if comment is None:
+            return NO_CONTEXTS
+
+        match = re.match(r"^#\s*type:\s*\(([^#]*)\)\s*->", comment)
+        if not match:
+            return NO_CONTEXTS
+        params_comments = _split_comment_param_declaration(match.group(1))
+
+        # Find the specific param being investigated
+        index = all_params.index(param)
+        # If the number of parameters doesn't match length of type comment,
+        # ignore first parameter (assume it's self).
+        if len(params_comments) != len(all_params):
+            debug.warning(
+                "Comments length != Params length %s %s",
+                params_comments, all_params
+            )
+        from jedi.evaluate.context.instance import InstanceArguments
+        if isinstance(execution_context.var_args, InstanceArguments):
+            if index == 0:
+                # Assume it's self, which is already handled
+                return NO_CONTEXTS
+            index -= 1
+        if index >= len(params_comments):
+            return NO_CONTEXTS
+
+        param_comment = params_comments[index]
+        return _evaluate_annotation_string(
+            execution_context.function_context.get_default_param_context(),
+            param_comment
+        )
+    # Annotations are like default params and resolve in the same way.
+    context = execution_context.function_context.get_default_param_context()
+    return eval_annotation(context, annotation)
+
+
+def py__annotations__(funcdef):
+    dct = {}
+    for function_param in funcdef.get_params():
+        param_annotation = function_param.annotation
+        if param_annotation is not None:
+            dct[function_param.name.value] = param_annotation
+
+    return_annotation = funcdef.annotation
+    if return_annotation:
+        dct['return'] = return_annotation
+    return dct
+
+
+@evaluator_method_cache()
+def infer_return_types(function_execution_context):
+    """
+    Infers the type of a function's return value,
+    according to type annotations.
+    """
+    all_annotations = py__annotations__(function_execution_context.tree_node)
+    annotation = all_annotations.get("return", None)
+    if annotation is None:
+        # If there is no Python 3-type annotation, look for a Python 2-type annotation
+        node = function_execution_context.tree_node
+        comment = parser_utils.get_following_comment_same_line(node)
+        if comment is None:
+            return NO_CONTEXTS
+
+        match = re.match(r"^#\s*type:\s*\([^#]*\)\s*->\s*([^#]*)", comment)
+        if not match:
+            return NO_CONTEXTS
+
+        return _evaluate_annotation_string(
+            function_execution_context.function_context.get_default_param_context(),
+            match.group(1).strip()
+        ).execute_annotation()
+        if annotation is None:
+            return NO_CONTEXTS
+
+    context = function_execution_context.function_context.get_default_param_context()
+    unknown_type_vars = list(find_unknown_type_vars(context, annotation))
+    annotation_contexts = eval_annotation(context, annotation)
+    if not unknown_type_vars:
+        return annotation_contexts.execute_annotation()
+
+    type_var_dict = infer_type_vars_for_execution(function_execution_context, all_annotations)
+
+    return ContextSet.from_sets(
+        ann.define_generics(type_var_dict)
+        if isinstance(ann, (AbstractAnnotatedClass, TypeVar)) else ContextSet({ann})
+        for ann in annotation_contexts
+    ).execute_annotation()
+
+
+def infer_type_vars_for_execution(execution_context, annotation_dict):
+    """
+    Some functions use type vars that are not defined by the class, but rather
+    only defined in the function. See for example `iter`. In those cases we
+    want to:
+
+    1. Search for undefined type vars.
+    2. Infer type vars with the execution state we have.
+    3. Return the union of all type vars that have been found.
+    """
+    context = execution_context.function_context.get_default_param_context()
+
+    annotation_variable_results = {}
+    executed_params, _ = execution_context.get_executed_params_and_issues()
+    for executed_param in executed_params:
+        try:
+            annotation_node = annotation_dict[executed_param.string_name]
+        except KeyError:
+            continue
+
+        annotation_variables = find_unknown_type_vars(context, annotation_node)
+        if annotation_variables:
+            # Infer unknown type var
+            annotation_context_set = context.eval_node(annotation_node)
+            star_count = executed_param._param_node.star_count
+            actual_context_set = executed_param.infer(use_hints=False)
+            if star_count == 1:
+                actual_context_set = actual_context_set.merge_types_of_iterate()
+            elif star_count == 2:
+                # TODO _dict_values is not public.
+                actual_context_set = actual_context_set.try_merge('_dict_values')
+            for ann in annotation_context_set:
+                _merge_type_var_dicts(
+                    annotation_variable_results,
+                    _infer_type_vars(ann, actual_context_set),
+                )
+
+    return annotation_variable_results
+
+
+def _merge_type_var_dicts(base_dict, new_dict):
+    for type_var_name, contexts in new_dict.items():
+        try:
+            base_dict[type_var_name] |= contexts
+        except KeyError:
+            base_dict[type_var_name] = contexts
+
+
+def _infer_type_vars(annotation_context, context_set):
+    """
+    This function tries to find information about undefined type vars and
+    returns a dict from type var name to context set.
+
+    This is for example important to understand what `iter([1])` returns.
+    According to typeshed, `iter` returns an `Iterator[_T]`:
+
+        def iter(iterable: Iterable[_T]) -> Iterator[_T]: ...
+
+    This functions would generate `int` for `_T` in this case, because it
+    unpacks the `Iterable`.
+    """
+    type_var_dict = {}
+    if isinstance(annotation_context, TypeVar):
+        return {annotation_context.py__name__(): context_set.py__class__()}
+    elif isinstance(annotation_context, LazyGenericClass):
+        name = annotation_context.py__name__()
+        if name == 'Iterable':
+            given = annotation_context.get_generics()
+            if given:
+                for nested_annotation_context in given[0]:
+                    _merge_type_var_dicts(
+                        type_var_dict,
+                        _infer_type_vars(
+                            nested_annotation_context,
+                            context_set.merge_types_of_iterate()
+                        )
+                    )
+        elif name == 'Mapping':
+            given = annotation_context.get_generics()
+            if len(given) == 2:
+                for context in context_set:
+                    try:
+                        method = context.get_mapping_item_contexts
+                    except AttributeError:
+                        continue
+                    key_contexts, value_contexts = method()
+
+                    for nested_annotation_context in given[0]:
+                        _merge_type_var_dicts(
+                            type_var_dict,
+                            _infer_type_vars(
+                                nested_annotation_context,
+                                key_contexts,
+                            )
+                        )
+                    for nested_annotation_context in given[1]:
+                        _merge_type_var_dicts(
+                            type_var_dict,
+                            _infer_type_vars(
+                                nested_annotation_context,
+                                value_contexts,
+                            )
+                        )
+    return type_var_dict
+
+
+def find_type_from_comment_hint_for(context, node, name):
+    return _find_type_from_comment_hint(context, node, node.children[1], name)
+
+
+def find_type_from_comment_hint_with(context, node, name):
+    assert len(node.children[1].children) == 3, \
+        "Can only be here when children[1] is 'foo() as f'"
+    varlist = node.children[1].children[2]
+    return _find_type_from_comment_hint(context, node, varlist, name)
+
+
+def find_type_from_comment_hint_assign(context, node, name):
+    return _find_type_from_comment_hint(context, node, node.children[0], name)
+
+
+def _find_type_from_comment_hint(context, node, varlist, name):
+    index = None
+    if varlist.type in ("testlist_star_expr", "exprlist", "testlist"):
+        # something like "a, b = 1, 2"
+        index = 0
+        for child in varlist.children:
+            if child == name:
+                break
+            if child.type == "operator":
+                continue
+            index += 1
+        else:
+            return []
+
+    comment = parser_utils.get_following_comment_same_line(node)
+    if comment is None:
+        return []
+    match = re.match(r"^#\s*type:\s*([^#]*)", comment)
+    if match is None:
+        return []
+    return _evaluate_annotation_string(
+        context, match.group(1).strip(), index
+    ).execute_annotation()
+
+
+def find_unknown_type_vars(context, node):
+    def check_node(node):
+        if node.type in ('atom_expr', 'power'):
+            trailer = node.children[-1]
+            if trailer.type == 'trailer' and trailer.children[0] == '[':
+                for subscript_node in _unpack_subscriptlist(trailer.children[1]):
+                    check_node(subscript_node)
+        else:
+            type_var_set = context.eval_node(node)
+            for type_var in type_var_set:
+                if isinstance(type_var, TypeVar) and type_var not in found:
+                    found.append(type_var)
+
+    found = []  # We're not using a set, because the order matters.
+    check_node(node)
+    return found
+
+
+def _unpack_subscriptlist(subscriptlist):
+    if subscriptlist.type == 'subscriptlist':
+        for subscript in subscriptlist.children[::2]:
+            if subscript.type != 'subscript':
+                yield subscript
+    else:
+        if subscriptlist.type != 'subscript':
+            yield subscriptlist

jedi/evaluate/gradual/conversion.py

@@ -0,0 +1,199 @@
+from jedi import debug
+from jedi.evaluate.base_context import ContextSet, \
+    NO_CONTEXTS
+from jedi.evaluate.utils import to_list
+from jedi.evaluate.gradual.stub_context import StubModuleContext
+
+
+def _stub_to_python_context_set(stub_context, ignore_compiled=False):
+    stub_module = stub_context.get_root_context()
+    if not stub_module.is_stub():
+        return ContextSet([stub_context])
+
+    was_instance = stub_context.is_instance()
+    if was_instance:
+        stub_context = stub_context.py__class__()
+
+    qualified_names = stub_context.get_qualified_names()
+    if qualified_names is None:
+        return NO_CONTEXTS
+
+    was_bound_method = stub_context.is_bound_method()
+    if was_bound_method:
+        # Infer the object first. We can infer the method later.
+        method_name = qualified_names[-1]
+        qualified_names = qualified_names[:-1]
+        was_instance = True
+
+    contexts = _infer_from_stub(stub_module, qualified_names, ignore_compiled)
+    if was_instance:
+        contexts = ContextSet.from_sets(
+            c.execute_evaluated()
+            for c in contexts
+            if c.is_class()
+        )
+    if was_bound_method:
+        # Now that the instance has been properly created, we can simply get
+        # the method.
+        contexts = contexts.py__getattribute__(method_name)
+    return contexts
+
+
+def _infer_from_stub(stub_module, qualified_names, ignore_compiled):
+    from jedi.evaluate.compiled.mixed import MixedObject
+    assert isinstance(stub_module, (StubModuleContext, MixedObject)), stub_module
+    non_stubs = stub_module.non_stub_context_set
+    if ignore_compiled:
+        non_stubs = non_stubs.filter(lambda c: not c.is_compiled())
+    for name in qualified_names:
+        non_stubs = non_stubs.py__getattribute__(name)
+    return non_stubs
+
+
+@to_list
+def _try_stub_to_python_names(names, prefer_stub_to_compiled=False):
+    for name in names:
+        module = name.get_root_context()
+        if not module.is_stub():
+            yield name
+            continue
+
+        name_list = name.get_qualified_names()
+        if name_list is None:
+            contexts = NO_CONTEXTS
+        else:
+            contexts = _infer_from_stub(
+                module,
+                name_list[:-1],
+                ignore_compiled=prefer_stub_to_compiled,
+            )
+        if contexts and name_list:
+            new_names = contexts.py__getattribute__(name_list[-1], is_goto=True)
+            for new_name in new_names:
+                yield new_name
+            if new_names:
+                continue
+        elif contexts:
+            for c in contexts:
+                yield c.name
+            continue
+        # This is the part where if we haven't found anything, just return the
+        # stub name.
+        yield name
+
+
+def _load_stub_module(module):
+    if module.is_stub():
+        return module
+    from jedi.evaluate.gradual.typeshed import _try_to_load_stub_cached
+    return _try_to_load_stub_cached(
+        module.evaluator,
+        import_names=module.string_names,
+        python_context_set=ContextSet([module]),
+        parent_module_context=None,
+        sys_path=module.evaluator.get_sys_path(),
+    )
+
+
+@to_list
+def _python_to_stub_names(names, fallback_to_python=False):
+    for name in names:
+        module = name.get_root_context()
+        if module.is_stub():
+            yield name
+            continue
+
+        if name.is_import():
+            for new_name in name.goto():
+                # Imports don't need to be converted, because they are already
+                # stubs if possible.
+                if fallback_to_python or new_name.is_stub():
+                    yield new_name
+            continue
+
+        name_list = name.get_qualified_names()
+        stubs = NO_CONTEXTS
+        if name_list is not None:
+            stub_module = _load_stub_module(module)
+            if stub_module is not None:
+                stubs = ContextSet({stub_module})
+                for name in name_list[:-1]:
+                    stubs = stubs.py__getattribute__(name)
+        if stubs and name_list:
+            new_names = stubs.py__getattribute__(name_list[-1], is_goto=True)
+            for new_name in new_names:
+                yield new_name
+            if new_names:
+                continue
+        elif stubs:
+            for c in stubs:
+                yield c.name
+            continue
+        if fallback_to_python:
+            # This is the part where if we haven't found anything, just return
+            # the stub name.
+            yield name
+
+
+def convert_names(names, only_stubs=False, prefer_stubs=False):
+    assert not (only_stubs and prefer_stubs)
+    with debug.increase_indent_cm('convert names'):
+        if only_stubs or prefer_stubs:
+            return _python_to_stub_names(names, fallback_to_python=prefer_stubs)
+        else:
+            return _try_stub_to_python_names(names, prefer_stub_to_compiled=True)
+
+
+def convert_contexts(contexts, only_stubs=False, prefer_stubs=False, ignore_compiled=True):
+    assert not (only_stubs and prefer_stubs)
+    with debug.increase_indent_cm('convert contexts'):
+        if only_stubs or prefer_stubs:
+            return ContextSet.from_sets(
+                to_stub(context)
+                or (ContextSet({context}) if prefer_stubs else NO_CONTEXTS)
+                for context in contexts
+            )
+        else:
+            return ContextSet.from_sets(
+                _stub_to_python_context_set(stub_context, ignore_compiled=ignore_compiled)
+                or ContextSet({stub_context})
+                for stub_context in contexts
+            )
+
+
+# TODO merge with _python_to_stub_names?
+def to_stub(context):
+    if context.is_stub():
+        return ContextSet([context])
+
+    was_instance = context.is_instance()
+    if was_instance:
+        context = context.py__class__()
+
+    qualified_names = context.get_qualified_names()
+    stub_module = _load_stub_module(context.get_root_context())
+    if stub_module is None or qualified_names is None:
+        return NO_CONTEXTS
+
+    was_bound_method = context.is_bound_method()
+    if was_bound_method:
+        # Infer the object first. We can infer the method later.
+        method_name = qualified_names[-1]
+        qualified_names = qualified_names[:-1]
+        was_instance = True
+
+    stub_contexts = ContextSet([stub_module])
+    for name in qualified_names:
+        stub_contexts = stub_contexts.py__getattribute__(name)
+
+    if was_instance:
+        stub_contexts = ContextSet.from_sets(
+            c.execute_evaluated()
+            for c in stub_contexts
+            if c.is_class()
+        )
+    if was_bound_method:
+        # Now that the instance has been properly created, we can simply get
+        # the method.
+        stub_contexts = stub_contexts.py__getattribute__(method_name)
+    return stub_contexts

jedi/evaluate/gradual/stub_context.py

@@ -0,0 +1,105 @@
+from jedi.evaluate.base_context import ContextWrapper
+from jedi.evaluate.context.module import ModuleContext
+from jedi.evaluate.filters import ParserTreeFilter, \
+    TreeNameDefinition
+from jedi.evaluate.gradual.typing import TypingModuleFilterWrapper
+
+
+class StubModuleContext(ModuleContext):
+    def __init__(self, non_stub_context_set, *args, **kwargs):
+        super(StubModuleContext, self).__init__(*args, **kwargs)
+        self.non_stub_context_set = non_stub_context_set
+
+    def is_stub(self):
+        return True
+
+    def sub_modules_dict(self):
+        """
+        We have to overwrite this, because it's possible to have stubs that
+        don't have code for all the child modules. At the time of writing this
+        there are for example no stubs for `json.tool`.
+        """
+        names = {}
+        for context in self.non_stub_context_set:
+            try:
+                method = context.sub_modules_dict
+            except AttributeError:
+                pass
+            else:
+                names.update(method())
+        names.update(super(StubModuleContext, self).sub_modules_dict())
+        return names
+
+    def _get_first_non_stub_filters(self):
+        for context in self.non_stub_context_set:
+            yield next(context.get_filters(search_global=False))
+
+    def _get_stub_filters(self, search_global, **filter_kwargs):
+        return [StubFilter(
+            self.evaluator,
+            context=self,
+            search_global=search_global,
+            **filter_kwargs
+        )] + list(self.iter_star_filters(search_global=search_global))
+
+    def get_filters(self, search_global=False, until_position=None,
+                    origin_scope=None, **kwargs):
+        filters = super(StubModuleContext, self).get_filters(
+            search_global, until_position, origin_scope, **kwargs
+        )
+        next(filters)  # Ignore the first filter and replace it with our own
+        stub_filters = self._get_stub_filters(
+            search_global=search_global,
+            until_position=until_position,
+            origin_scope=origin_scope,
+        )
+        for f in stub_filters:
+            yield f
+
+        for f in filters:
+            yield f
+
+
+class TypingModuleWrapper(StubModuleContext):
+    def get_filters(self, *args, **kwargs):
+        filters = super(TypingModuleWrapper, self).get_filters(*args, **kwargs)
+        yield TypingModuleFilterWrapper(next(filters))
+        for f in filters:
+            yield f
+
+
+# From here on down we make looking up the sys.version_info fast.
+class _StubName(TreeNameDefinition):
+    def infer(self):
+        inferred = super(_StubName, self).infer()
+        if self.string_name == 'version_info' and self.get_root_context().py__name__() == 'sys':
+            return [VersionInfo(c) for c in inferred]
+        return inferred
+
+
+class StubFilter(ParserTreeFilter):
+    name_class = _StubName
+
+    def __init__(self, *args, **kwargs):
+        self._search_global = kwargs.pop('search_global')  # Python 2 :/
+        super(StubFilter, self).__init__(*args, **kwargs)
+
+    def _is_name_reachable(self, name):
+        if not super(StubFilter, self)._is_name_reachable(name):
+            return False
+
+        if not self._search_global:
+            # Imports in stub files are only public if they have an "as"
+            # export.
+            definition = name.get_definition()
+            if definition.type in ('import_from', 'import_name'):
+                if name.parent.type not in ('import_as_name', 'dotted_as_name'):
+                    return False
+            n = name.value
+            if n.startswith('_') and not (n.startswith('__') and n.endswith('__')):
+                return False
+        return True
+
+
+class VersionInfo(ContextWrapper):
+    pass

jedi/evaluate/gradual/typeshed.py

@@ -0,0 +1,289 @@
+import os
+import re
+from functools import wraps
+
+from jedi.file_io import FileIO
+from jedi._compatibility import FileNotFoundError, cast_path
+from jedi.parser_utils import get_cached_code_lines
+from jedi.evaluate.base_context import ContextSet, NO_CONTEXTS
+from jedi.evaluate.gradual.stub_context import TypingModuleWrapper, StubModuleContext
+
+_jedi_path = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+TYPESHED_PATH = os.path.join(_jedi_path, 'third_party', 'typeshed')
+
+_IMPORT_MAP = dict(
+    _collections='collections',
+    _socket='socket',
+)
+
+
+def _merge_create_stub_map(directories):
+    map_ = {}
+    for directory in directories:
+        map_.update(_create_stub_map(directory))
+    return map_
+
+
+def _create_stub_map(directory):
+    """
+    Create a mapping of an importable name in Python to a stub file.
+    """
+    def generate():
+        try:
+            listed = os.listdir(directory)
+        except (FileNotFoundError, OSError):
+            # OSError is Python 2
+            return
+
+        for entry in listed:
+            entry = cast_path(entry)
+            path = os.path.join(directory, entry)
+            if os.path.isdir(path):
+                init = os.path.join(path, '__init__.pyi')
+                if os.path.isfile(init):
+                    yield entry, init
+            elif entry.endswith('.pyi') and os.path.isfile(path):
+                name = entry.rstrip('.pyi')
+                if name != '__init__':
+                    yield name, path
+
+    # Create a dictionary from the tuple generator.
+    return dict(generate())
+
+
+def _get_typeshed_directories(version_info):
+    check_version_list = ['2and3', str(version_info.major)]
+    for base in ['stdlib', 'third_party']:
+        base = os.path.join(TYPESHED_PATH, base)
+        base_list = os.listdir(base)
+        for base_list_entry in base_list:
+            match = re.match(r'(\d+)\.(\d+)$', base_list_entry)
+            if match is not None:
+                if int(match.group(1)) == version_info.major \
+                        and int(match.group(2)) <= version_info.minor:
+                    check_version_list.append(base_list_entry)
+
+        for check_version in check_version_list:
+            yield os.path.join(base, check_version)
+
+
+_version_cache = {}
+
+
+def _cache_stub_file_map(version_info):
+    """
+    Returns a map of an importable name in Python to a stub file.
+    """
+    # TODO this caches the stub files indefinitely, maybe use a time cache
+    # for that?
+    version = version_info[:2]
+    try:
+        return _version_cache[version]
+    except KeyError:
+        pass
+
+    _version_cache[version] = file_set = \
+        _merge_create_stub_map(_get_typeshed_directories(version_info))
+    return file_set
+
+
+def import_module_decorator(func):
+    @wraps(func)
+    def wrapper(evaluator, import_names, parent_module_context, sys_path, prefer_stubs):
+        try:
+            python_context_set = evaluator.module_cache.get(import_names)
+        except KeyError:
+            if parent_module_context is not None and parent_module_context.is_stub():
+                parent_module_contexts = parent_module_context.non_stub_context_set
+            else:
+                parent_module_contexts = [parent_module_context]
+            if import_names == ('os', 'path'):
+                # This is a huge exception, we follow a nested import
+                # ``os.path``, because it's a very important one in Python
+                # that is being achieved by messing with ``sys.modules`` in
+                # ``os``.
+                python_parent = next(iter(parent_module_contexts))
+                if python_parent is None:
+                    python_parent, = evaluator.import_module(('os',), prefer_stubs=False)
+                python_context_set = python_parent.py__getattribute__('path')
+            else:
+                python_context_set = ContextSet.from_sets(
+                    func(evaluator, import_names, p, sys_path,)
+                    for p in parent_module_contexts
+                )
+            evaluator.module_cache.add(import_names, python_context_set)
+
+        if not prefer_stubs:
+            return python_context_set
+
+        stub = _try_to_load_stub_cached(evaluator, import_names, python_context_set,
+                                        parent_module_context, sys_path)
+        if stub is not None:
+            return ContextSet([stub])
+        return python_context_set
+
+    return wrapper
+
+
+def _try_to_load_stub_cached(evaluator, import_names, *args, **kwargs):
+    try:
+        return evaluator.stub_module_cache[import_names]
+    except KeyError:
+        pass
+
+    # TODO is this needed? where are the exceptions coming from that make this
+    # necessary? Just remove this line.
+    evaluator.stub_module_cache[import_names] = None
+    evaluator.stub_module_cache[import_names] = result = \
+        _try_to_load_stub(evaluator, import_names, *args, **kwargs)
+    return result
+
+
+def _try_to_load_stub(evaluator, import_names, python_context_set,
+                      parent_module_context, sys_path):
+    """
+    Trying to load a stub for a set of import_names.
+
+    This is modelled to work like "PEP 561 -- Distributing and Packaging Type
+    Information", see https://www.python.org/dev/peps/pep-0561.
+    """
+    if parent_module_context is None and len(import_names) > 1:
+        try:
+            parent_module_context = _try_to_load_stub_cached(
+                evaluator, import_names[:-1], NO_CONTEXTS,
+                parent_module_context=None, sys_path=sys_path)
+        except KeyError:
+            pass
+
+    # 1. Try to load foo-stubs folders on path for import name foo.
+    if len(import_names) == 1:
+        # foo-stubs
+        for p in sys_path:
+            init = os.path.join(p, *import_names) + '-stubs' + os.path.sep + '__init__.pyi'
+            m = _try_to_load_stub_from_file(
+                evaluator,
+                python_context_set,
+                file_io=FileIO(init),
+                import_names=import_names,
+            )
+            if m is not None:
+                return m
+
+    # 2. Try to load pyi files next to py files.
+    for c in python_context_set:
+        try:
+            method = c.py__file__
+        except AttributeError:
+            pass
+        else:
+            file_path = method()
+            file_paths = []
+            if c.is_namespace():
+                file_paths = [os.path.join(p, '__init__.pyi') for p in c.py__path__()]
+            elif file_path is not None and file_path.endswith('.py'):
+                file_paths = [file_path + 'i']
+
+            for file_path in file_paths:
+                m = _try_to_load_stub_from_file(
+                    evaluator,
+                    python_context_set,
+                    # The file path should end with .pyi
+                    file_io=FileIO(file_path),
+                    import_names=import_names,
+                )
+                if m is not None:
+                    return m
+
+    # 3. Try to load typeshed
+    m = _load_from_typeshed(evaluator, python_context_set, parent_module_context, import_names)
+    if m is not None:
+        return m
+
+    # 4. Try to load pyi file somewhere if python_context_set was not defined.
+    if not python_context_set:
+        if parent_module_context is not None:
+            try:
+                method = parent_module_context.py__path__
+            except AttributeError:
+                check_path = []
+            else:
+                check_path = method()
+            # In case import_names
+            names_for_path = (import_names[-1],)
+        else:
+            check_path = sys_path
+            names_for_path = import_names
+
+        for p in check_path:
+            m = _try_to_load_stub_from_file(
+                evaluator,
+                python_context_set,
+                file_io=FileIO(os.path.join(p, *names_for_path) + '.pyi'),
+                import_names=import_names,
+            )
+            if m is not None:
+                return m
+
+    # If no stub is found, that's fine, the calling function has to deal with
+    # it.
+    return None
+
+
+def _load_from_typeshed(evaluator, python_context_set, parent_module_context, import_names):
+    import_name = import_names[-1]
+    map_ = None
+    if len(import_names) == 1:
+        map_ = _cache_stub_file_map(evaluator.grammar.version_info)
+        import_name = _IMPORT_MAP.get(import_name, import_name)
+    elif isinstance(parent_module_context, StubModuleContext):
+        if not parent_module_context.is_package:
+            # Only if it's a package (= a folder) something can be
+            # imported.
+            return None
+        path = parent_module_context.py__path__()
+        map_ = _merge_create_stub_map(path)
+
+    if map_ is not None:
+        path = map_.get(import_name)
+        if path is not None:
+            return _try_to_load_stub_from_file(
+                evaluator,
+                python_context_set,
+                file_io=FileIO(path),
+                import_names=import_names,
+            )
+
+
+def _try_to_load_stub_from_file(evaluator, python_context_set, file_io, import_names):
+    try:
+        stub_module_node = evaluator.parse(
+            file_io=file_io,
+            cache=True,
+            use_latest_grammar=True
+        )
+    except (OSError, IOError):  # IOError is Python 2 only
+        # The file that you're looking for doesn't exist (anymore).
+        return None
+    else:
+        return create_stub_module(
+            evaluator, python_context_set, stub_module_node, file_io,
+            import_names
+        )
+
+
+def create_stub_module(evaluator, python_context_set, stub_module_node, file_io, import_names):
+    if import_names == ('typing',):
+        module_cls = TypingModuleWrapper
+    else:
+        module_cls = StubModuleContext
+    file_name = os.path.basename(file_io.path)
+    stub_module_context = module_cls(
+        python_context_set, evaluator, stub_module_node,
+        file_io=file_io,
+        string_names=import_names,
+        # The code was loaded with latest_grammar, so use
+        # that.
+        code_lines=get_cached_code_lines(evaluator.latest_grammar, file_io.path),
+        is_package=file_name == '__init__.pyi',
+    )
+    return stub_module_context

jedi/evaluate/gradual/typing.py

@@ -0,0 +1,707 @@
+"""
+We need to somehow work with the typing objects. Since the typing objects are
+pretty bare we need to add all the Jedi customizations to make them work as
+contexts.
+
+This file deals with all the typing.py cases.
+"""
+from jedi._compatibility import unicode, force_unicode
+from jedi import debug
+from jedi.evaluate.cache import evaluator_method_cache
+from jedi.evaluate.compiled import builtin_from_name
+from jedi.evaluate.base_context import ContextSet, NO_CONTEXTS, Context, \
+    iterator_to_context_set, ContextWrapper, LazyContextWrapper
+from jedi.evaluate.lazy_context import LazyKnownContexts
+from jedi.evaluate.context.iterable import SequenceLiteralContext
+from jedi.evaluate.arguments import repack_with_argument_clinic
+from jedi.evaluate.utils import to_list
+from jedi.evaluate.filters import FilterWrapper
+from jedi.evaluate.names import NameWrapper, AbstractTreeName, \
+    AbstractNameDefinition, ContextName
+from jedi.evaluate.helpers import is_string
+from jedi.evaluate.context.klass import ClassMixin, ClassFilter
+
+_PROXY_CLASS_TYPES = 'Tuple Generic Protocol Callable Type'.split()
+_TYPE_ALIAS_TYPES = {
+    'List': 'builtins.list',
+    'Dict': 'builtins.dict',
+    'Set': 'builtins.set',
+    'FrozenSet': 'builtins.frozenset',
+    'ChainMap': 'collections.ChainMap',
+    'Counter': 'collections.Counter',
+    'DefaultDict': 'collections.defaultdict',
+    'Deque': 'collections.deque',
+}
+_PROXY_TYPES = 'Optional Union ClassVar'.split()
+
+
+class TypingName(AbstractTreeName):
+    def __init__(self, context, other_name):
+        super(TypingName, self).__init__(context.parent_context, other_name.tree_name)
+        self._context = context
+
+    def infer(self):
+        return ContextSet([self._context])
+
+
+class _BaseTypingContext(Context):
+    def __init__(self, evaluator, parent_context, tree_name):
+        super(_BaseTypingContext, self).__init__(evaluator, parent_context)
+        self._tree_name = tree_name
+
+    @property
+    def tree_node(self):
+        return self._tree_name
+
+    def get_filters(self, *args, **kwargs):
+        # TODO this is obviously wrong. Is it though?
+        class EmptyFilter(ClassFilter):
+            def __init__(self):
+                pass
+
+            def get(self, name, **kwargs):
+                return []
+
+            def values(self, **kwargs):
+                return []
+
+        yield EmptyFilter()
+
+    def py__class__(self):
+        # TODO this is obviously not correct, but at least gives us a class if
+        # we have none. Some of these objects don't really have a base class in
+        # typeshed.
+        return builtin_from_name(self.evaluator, u'object')
+
+    @property
+    def name(self):
+        return ContextName(self, self._tree_name)
+
+    def __repr__(self):
+        return '%s(%s)' % (self.__class__.__name__, self._tree_name.value)
+
+
+class TypingModuleName(NameWrapper):
+    def infer(self):
+        return ContextSet(self._remap())
+
+    def _remap(self):
+        name = self.string_name
+        evaluator = self.parent_context.evaluator
+        try:
+            actual = _TYPE_ALIAS_TYPES[name]
+        except KeyError:
+            pass
+        else:
+            yield TypeAlias.create_cached(evaluator, self.parent_context, self.tree_name, actual)
+            return
+
+        if name in _PROXY_CLASS_TYPES:
+            yield TypingClassContext.create_cached(evaluator, self.parent_context, self.tree_name)
+        elif name in _PROXY_TYPES:
+            yield TypingContext.create_cached(evaluator, self.parent_context, self.tree_name)
+        elif name == 'runtime':
+            # We don't want anything here, not sure what this function is
+            # supposed to do, since it just appears in the stubs and shouldn't
+            # have any effects there (because it's never executed).
+            return
+        elif name == 'TypeVar':
+            yield TypeVarClass.create_cached(evaluator, self.parent_context, self.tree_name)
+        elif name == 'Any':
+            yield Any.create_cached(evaluator, self.parent_context, self.tree_name)
+        elif name == 'TYPE_CHECKING':
+            # This is needed for e.g. imports that are only available for type
+            # checking or are in cycles. The user can then check this variable.
+            yield builtin_from_name(evaluator, u'True')
+        elif name == 'overload':
+            yield OverloadFunction.create_cached(evaluator, self.parent_context, self.tree_name)
+        elif name == 'NewType':
+            yield NewTypeFunction.create_cached(evaluator, self.parent_context, self.tree_name)
+        elif name == 'cast':
+            # TODO implement cast
+            yield CastFunction.create_cached(evaluator, self.parent_context, self.tree_name)
+        elif name == 'TypedDict':
+            # TODO doesn't even exist in typeshed/typing.py, yet. But will be
+            # added soon.
+            pass
+        elif name in ('no_type_check', 'no_type_check_decorator'):
+            # This is not necessary, as long as we are not doing type checking.
+            for c in self._wrapped_name.infer():  # Fuck my life Python 2
+                yield c
+        else:
+            # Everything else shouldn't be relevant for type checking.
+            for c in self._wrapped_name.infer():  # Fuck my life Python 2
+                yield c
+
+
+class TypingModuleFilterWrapper(FilterWrapper):
+    name_wrapper_class = TypingModuleName
+
+
+class _WithIndexBase(_BaseTypingContext):
+    def __init__(self, evaluator, parent_context, name, index_context, context_of_index):
+        super(_WithIndexBase, self).__init__(evaluator, parent_context, name)
+        self._index_context = index_context
+        self._context_of_index = context_of_index
+
+    def __repr__(self):
+        return '<%s: %s[%s]>' % (
+            self.__class__.__name__,
+            self._tree_name.value,
+            self._index_context,
+        )
+
+
+class TypingContextWithIndex(_WithIndexBase):
+    def execute_annotation(self):
+        string_name = self._tree_name.value
+
+        if string_name == 'Union':
+            # This is kind of a special case, because we have Unions (in Jedi
+            # ContextSets).
+            return self.gather_annotation_classes().execute_annotation()
+        elif string_name == 'Optional':
+            # Optional is basically just saying it's either None or the actual
+            # type.
+            return self.gather_annotation_classes().execute_annotation() \
+                | ContextSet([builtin_from_name(self.evaluator, u'None')])
+        elif string_name == 'Type':
+            # The type is actually already given in the index_context
+            return ContextSet([self._index_context])
+        elif string_name == 'ClassVar':
+            # For now don't do anything here, ClassVars are always used.
+            return self._index_context.execute_annotation()
+
+        cls = globals()[string_name]
+        return ContextSet([cls(
+            self.evaluator,
+            self.parent_context,
+            self._tree_name,
+            self._index_context,
+            self._context_of_index
+        )])
+
+    def gather_annotation_classes(self):
+        return ContextSet.from_sets(
+            _iter_over_arguments(self._index_context, self._context_of_index)
+        )
+
+
+class TypingContext(_BaseTypingContext):
+    index_class = TypingContextWithIndex
+    py__simple_getitem__ = None
+
+    def py__getitem__(self, index_context_set, contextualized_node):
+        return ContextSet(
+            self.index_class.create_cached(
+                self.evaluator,
+                self.parent_context,
+                self._tree_name,
+                index_context,
+                context_of_index=contextualized_node.context)
+            for index_context in index_context_set
+        )
+
+
+class _TypingClassMixin(object):
+    def py__bases__(self):
+        return [LazyKnownContexts(
+            self.evaluator.builtins_module.py__getattribute__('object')
+        )]
+
+    def get_metaclasses(self):
+        return []
+
+
+class TypingClassContextWithIndex(_TypingClassMixin, TypingContextWithIndex, ClassMixin):
+    pass
+
+
+class TypingClassContext(_TypingClassMixin, TypingContext, ClassMixin):
+    index_class = TypingClassContextWithIndex
+
+
+def _iter_over_arguments(maybe_tuple_context, defining_context):
+    def iterate():
+        if isinstance(maybe_tuple_context, SequenceLiteralContext):
+            for lazy_context in maybe_tuple_context.py__iter__(contextualized_node=None):
+                yield lazy_context.infer()
+        else:
+            yield ContextSet([maybe_tuple_context])
+
+    def resolve_forward_references(context_set):
+        for context in context_set:
+            if is_string(context):
+                from jedi.evaluate.gradual.annotation import _get_forward_reference_node
+                node = _get_forward_reference_node(defining_context, context.get_safe_value())
+                if node is not None:
+                    for c in defining_context.eval_node(node):
+                        yield c
+            else:
+                yield context
+
+    for context_set in iterate():
+        yield ContextSet(resolve_forward_references(context_set))
+
+
+class TypeAlias(LazyContextWrapper):
+    def __init__(self, parent_context, origin_tree_name, actual):
+        self.evaluator = parent_context.evaluator
+        self.parent_context = parent_context
+        self._origin_tree_name = origin_tree_name
+        self._actual = actual  # e.g. builtins.list
+
+    @property
+    def name(self):
+        return ContextName(self, self._origin_tree_name)
+
+    def py__name__(self):
+        return self.name.string_name
+
+    def __repr__(self):
+        return '<%s: %s>' % (self.__class__.__name__, self._actual)
+
+    def _get_wrapped_context(self):
+        module_name, class_name = self._actual.split('.')
+        if self.evaluator.environment.version_info.major == 2 and module_name == 'builtins':
+            module_name = '__builtin__'
+
+        # TODO use evaluator.import_module?
+        from jedi.evaluate.imports import Importer
+        module, = Importer(
+            self.evaluator, [module_name], self.evaluator.builtins_module
+        ).follow()
+        classes = module.py__getattribute__(class_name)
+        # There should only be one, because it's code that we control.
+        assert len(classes) == 1, classes
+        cls = next(iter(classes))
+        return cls
+
+
+class _ContainerBase(_WithIndexBase):
+    def _get_getitem_contexts(self, index):
+        args = _iter_over_arguments(self._index_context, self._context_of_index)
+        for i, contexts in enumerate(args):
+            if i == index:
+                return contexts
+
+        debug.warning('No param #%s found for annotation %s', index, self._index_context)
+        return NO_CONTEXTS
+
+
+class Callable(_ContainerBase):
+    def py__call__(self, arguments):
+        # The 0th index are the arguments.
+        return self._get_getitem_contexts(1).execute_annotation()
+
+
+class Tuple(_ContainerBase):
+    def _is_homogenous(self):
+        # To specify a variable-length tuple of homogeneous type, Tuple[T, ...]
+        # is used.
+        if isinstance(self._index_context, SequenceLiteralContext):
+            entries = self._index_context.get_tree_entries()
+            if len(entries) == 2 and entries[1] == '...':
+                return True
+        return False
+
+    def py__simple_getitem__(self, index):
+        if self._is_homogenous():
+            return self._get_getitem_contexts(0).execute_annotation()
+        else:
+            if isinstance(index, int):
+                return self._get_getitem_contexts(index).execute_annotation()
+
+            debug.dbg('The getitem type on Tuple was %s' % index)
+            return NO_CONTEXTS
+
+    def py__iter__(self, contextualized_node=None):
+        if self._is_homogenous():
+            yield LazyKnownContexts(self._get_getitem_contexts(0).execute_annotation())
+        else:
+            if isinstance(self._index_context, SequenceLiteralContext):
+                for i in range(self._index_context.py__len__()):
+                    yield LazyKnownContexts(self._get_getitem_contexts(i).execute_annotation())
+
+    def py__getitem__(self, index_context_set, contextualized_node):
+        if self._is_homogenous():
+            return self._get_getitem_contexts(0).execute_annotation()
+
+        return ContextSet.from_sets(
+            _iter_over_arguments(self._index_context, self._context_of_index)
+        ).execute_annotation()
+
+
+class Generic(_ContainerBase):
+    pass
+
+
+class Protocol(_ContainerBase):
+    pass
+
+
+class Any(_BaseTypingContext):
+    def execute_annotation(self):
+        debug.warning('Used Any - returned no results')
+        return NO_CONTEXTS
+
+
+class TypeVarClass(_BaseTypingContext):
+    def py__call__(self, arguments):
+        unpacked = arguments.unpack()
+
+        key, lazy_context = next(unpacked, (None, None))
+        var_name = self._find_string_name(lazy_context)
+        # The name must be given, otherwise it's useless.
+        if var_name is None or key is not None:
+            debug.warning('Found a variable without a name %s', arguments)
+            return NO_CONTEXTS
+
+        return ContextSet([TypeVar.create_cached(
+            self.evaluator,
+            self.parent_context,
+            self._tree_name,
+            var_name,
+            unpacked
+        )])
+
+    def _find_string_name(self, lazy_context):
+        if lazy_context is None:
+            return None
+
+        context_set = lazy_context.infer()
+        if not context_set:
+            return None
+        if len(context_set) > 1:
+            debug.warning('Found multiple contexts for a type variable: %s', context_set)
+
+        name_context = next(iter(context_set))
+        try:
+            method = name_context.get_safe_value
+        except AttributeError:
+            return None
+        else:
+            safe_value = method(default=None)
+            if self.evaluator.environment.version_info.major == 2:
+                if isinstance(safe_value, bytes):
+                    return force_unicode(safe_value)
+            if isinstance(safe_value, (str, unicode)):
+                return safe_value
+            return None
+
+
+class TypeVar(_BaseTypingContext):
+    def __init__(self, evaluator, parent_context, tree_name, var_name, unpacked_args):
+        super(TypeVar, self).__init__(evaluator, parent_context, tree_name)
+        self._var_name = var_name
+
+        self._constraints_lazy_contexts = []
+        self._bound_lazy_context = None
+        self._covariant_lazy_context = None
+        self._contravariant_lazy_context = None
+        for key, lazy_context in unpacked_args:
+            if key is None:
+                self._constraints_lazy_contexts.append(lazy_context)
+            else:
+                if key == 'bound':
+                    self._bound_lazy_context = lazy_context
+                elif key == 'covariant':
+                    self._covariant_lazy_context = lazy_context
+                elif key == 'contravariant':
+                    self._contra_variant_lazy_context = lazy_context
+                else:
+                    debug.warning('Invalid TypeVar param name %s', key)
+
+    def py__name__(self):
+        return self._var_name
+
+    def get_filters(self, *args, **kwargs):
+        return iter([])
+
+    def _get_classes(self):
+        if self._bound_lazy_context is not None:
+            return self._bound_lazy_context.infer()
+        if self._constraints_lazy_contexts:
+            return self.constraints
+        debug.warning('Tried to infer the TypeVar %s without a given type', self._var_name)
+        return NO_CONTEXTS
+
+    def is_same_class(self, other):
+        # Everything can match an undefined type var.
+        return True
+
+    @property
+    def constraints(self):
+        return ContextSet.from_sets(
+            lazy.infer() for lazy in self._constraints_lazy_contexts
+        )
+
+    def define_generics(self, type_var_dict):
+        try:
+            found = type_var_dict[self.py__name__()]
+        except KeyError:
+            pass
+        else:
+            if found:
+                return found
+        return self._get_classes() or ContextSet({self})
+
+    def execute_annotation(self):
+        return self._get_classes().execute_annotation()
+
+    def __repr__(self):
+        return '<%s: %s>' % (self.__class__.__name__, self.py__name__())
+
+
+class OverloadFunction(_BaseTypingContext):
+    @repack_with_argument_clinic('func, /')
+    def py__call__(self, func_context_set):
+        # Just pass arguments through.
+        return func_context_set
+
+
+class NewTypeFunction(_BaseTypingContext):
+    def py__call__(self, arguments):
+        ordered_args = arguments.unpack()
+        next(ordered_args, (None, None))
+        _, second_arg = next(ordered_args, (None, None))
+        if second_arg is None:
+            return NO_CONTEXTS
+        return ContextSet(
+            NewType(
+                self.evaluator,
+                contextualized_node.context,
+                contextualized_node.node,
+                second_arg.infer(),
+            ) for contextualized_node in arguments.get_calling_nodes())
+
+
+class NewType(Context):
+    def __init__(self, evaluator, parent_context, tree_node, type_context_set):
+        super(NewType, self).__init__(evaluator, parent_context)
+        self._type_context_set = type_context_set
+        self.tree_node = tree_node
+
+    def py__call__(self, arguments):
+        return self._type_context_set.execute_annotation()
+
+
+class CastFunction(_BaseTypingContext):
+    @repack_with_argument_clinic('type, object, /')
+    def py__call__(self, type_context_set, object_context_set):
+        return type_context_set.execute_annotation()
+
+
+class BoundTypeVarName(AbstractNameDefinition):
+    """
+    This type var was bound to a certain type, e.g. int.
+    """
+    def __init__(self, type_var, context_set):
+        self._type_var = type_var
+        self.parent_context = type_var.parent_context
+        self._context_set = context_set
+
+    def infer(self):
+        def iter_():
+            for context in self._context_set:
+                # Replace any with the constraints if they are there.
+                if isinstance(context, Any):
+                    for constraint in self._type_var.constraints:
+                        yield constraint
+                else:
+                    yield context
+        return ContextSet(iter_())
+
+    def py__name__(self):
+        return self._type_var.py__name__()
+
+    def __repr__(self):
+        return '<%s %s -> %s>' % (self.__class__.__name__, self.py__name__(), self._context_set)
+
+
+class TypeVarFilter(object):
+    """
+    A filter for all given variables in a class.
+
+        A = TypeVar('A')
+        B = TypeVar('B')
+        class Foo(Mapping[A, B]):
+            ...
+
+    In this example we would have two type vars given: A and B
+    """
+    def __init__(self, generics, type_vars):
+        self._generics = generics
+        self._type_vars = type_vars
+
+    def get(self, name):
+        for i, type_var in enumerate(self._type_vars):
+            if type_var.py__name__() == name:
+                try:
+                    return [BoundTypeVarName(type_var, self._generics[i])]
+                except IndexError:
+                    return [type_var.name]
+        return []
+
+    def values(self):
+        # The values are not relevant. If it's not searched exactly, the type
+        # vars are just global and should be looked up as that.
+        return []
+
+
+class AbstractAnnotatedClass(ClassMixin, ContextWrapper):
+    def get_type_var_filter(self):
+        return TypeVarFilter(self.get_generics(), self.list_type_vars())
+
+    def get_filters(self, search_global=False, *args, **kwargs):
+        filters = super(AbstractAnnotatedClass, self).get_filters(
+            search_global,
+            *args, **kwargs
+        )
+        for f in filters:
+            yield f
+
+        if search_global:
+            # The type vars can only be looked up if it's a global search and
+            # not a direct lookup on the class.
+            yield self.get_type_var_filter()
+
+    def is_same_class(self, other):
+        if not isinstance(other, AbstractAnnotatedClass):
+            return False
+
+        if self.tree_node != other.tree_node:
+            # TODO not sure if this is nice.
+            return False
+        given_params1 = self.get_generics()
+        given_params2 = other.get_generics()
+
+        if len(given_params1) != len(given_params2):
+            # If the amount of type vars doesn't match, the class doesn't
+            # match.
+            return False
+
+        # Now compare generics
+        return all(
+            any(
+                # TODO why is this ordering the correct one?
+                cls2.is_same_class(cls1)
+                for cls1 in class_set1
+                for cls2 in class_set2
+            ) for class_set1, class_set2 in zip(given_params1, given_params2)
+        )
+
+    def py__call__(self, arguments):
+        instance, = super(AbstractAnnotatedClass, self).py__call__(arguments)
+        return ContextSet([InstanceWrapper(instance)])
+
+    def get_generics(self):
+        raise NotImplementedError
+
+    def define_generics(self, type_var_dict):
+        changed = False
+        new_generics = []
+        for generic_set in self.get_generics():
+            contexts = NO_CONTEXTS
+            for generic in generic_set:
+                if isinstance(generic, (AbstractAnnotatedClass, TypeVar)):
+                    result = generic.define_generics(type_var_dict)
+                    contexts |= result
+                    if result != ContextSet({generic}):
+                        changed = True
+                else:
+                    contexts |= ContextSet([generic])
+            new_generics.append(contexts)
+
+        if not changed:
+            # There might not be any type vars that change. In that case just
+            # return itself, because it does not make sense to potentially lose
+            # cached results.
+            return ContextSet([self])
+
+        return ContextSet([GenericClass(
+            self._wrapped_context,
+            generics=tuple(new_generics)
+        )])
+
+    def __repr__(self):
+        return '<%s: %s%s>' % (
+            self.__class__.__name__,
+            self._wrapped_context,
+            list(self.get_generics()),
+        )
+
+    @to_list
+    def py__bases__(self):
+        for base in self._wrapped_context.py__bases__():
+            yield LazyAnnotatedBaseClass(self, base)
+
+
+class LazyGenericClass(AbstractAnnotatedClass):
+    def __init__(self, class_context, index_context, context_of_index):
+        super(LazyGenericClass, self).__init__(class_context)
+        self._index_context = index_context
+        self._context_of_index = context_of_index
+
+    @evaluator_method_cache()
+    def get_generics(self):
+        return list(_iter_over_arguments(self._index_context, self._context_of_index))
+
+
+class GenericClass(AbstractAnnotatedClass):
+    def __init__(self, class_context, generics):
+        super(GenericClass, self).__init__(class_context)
+        self._generics = generics
+
+    def get_generics(self):
+        return self._generics
+
+
+class LazyAnnotatedBaseClass(object):
+    def __init__(self, class_context, lazy_base_class):
+        self._class_context = class_context
+        self._lazy_base_class = lazy_base_class
+
+    @iterator_to_context_set
+    def infer(self):
+        for base in self._lazy_base_class.infer():
+            if isinstance(base, AbstractAnnotatedClass):
+                # Here we have to recalculate the given types.
+                yield GenericClass.create_cached(
+                    base.evaluator,
+                    base._wrapped_context,
+                    tuple(self._remap_type_vars(base)),
+                )
+            else:
+                yield base
+
+    def _remap_type_vars(self, base):
+        filter = self._class_context.get_type_var_filter()
+        for type_var_set in base.get_generics():
+            new = NO_CONTEXTS
+            for type_var in type_var_set:
+                if isinstance(type_var, TypeVar):
+                    names = filter.get(type_var.py__name__())
+                    new |= ContextSet.from_sets(
+                        name.infer() for name in names
+                    )
+                else:
+                    # Mostly will be type vars, except if in some cases
+                    # a concrete type will already be there. In that
+                    # case just add it to the context set.
+                    new |= ContextSet([type_var])
+            yield new
+
+
+class InstanceWrapper(ContextWrapper):
+    def py__stop_iteration_returns(self):
+        for cls in self._wrapped_context.class_context.py__mro__():
+            if cls.py__name__() == 'Generator':
+                generics = cls.get_generics()
+                try:
+                    return generics[2].execute_annotation()
+                except IndexError:
+                    pass
+            elif cls.py__name__() == 'Iterator':
+                return ContextSet([builtin_from_name(self.evaluator, u'None')])
+        return self._wrapped_context.py__stop_iteration_returns()

jedi/evaluate/gradual/utils.py

@@ -0,0 +1,32 @@
+import os
+
+from jedi.evaluate.gradual.typeshed import TYPESHED_PATH, create_stub_module
+
+
+def load_proper_stub_module(evaluator, file_io, import_names, module_node):
+    """
+    This function is given a random .pyi file and should return the proper
+    module.
+    """
+    path = file_io.path
+    assert path.endswith('.pyi')
+    if path.startswith(TYPESHED_PATH):
+        # /foo/stdlib/3/os/__init__.pyi -> stdlib/3/os/__init__
+        rest = path[len(TYPESHED_PATH) + 1: -4]
+        split_paths = tuple(rest.split(os.path.sep))
+        # Remove the stdlib/3 or third_party/3.5 part
+        import_names = split_paths[2:]
+        if import_names[-1] == '__init__':
+            import_names = import_names[:-1]
+
+    if import_names is not None:
+        actual_context_set = evaluator.import_module(import_names, prefer_stubs=False)
+        if not actual_context_set:
+            return None
+
+        stub = create_stub_module(
+            evaluator, actual_context_set, module_node, file_io, import_names
+        )
+        evaluator.stub_module_cache[import_names] = stub
+        return stub
+    return None

jedi/evaluate/helpers.py

@@ -1,8 +1,25 @@
 import copy
+import sys
+import re
+import os
 from itertools import chain
 from contextlib import contextmanager
 
-from jedi.parser import tree
+from parso.python import tree
+
+from jedi._compatibility import unicode
+from jedi.parser_utils import get_parent_scope
+
+
+def is_stdlib_path(path):
+    # Python standard library paths look like this:
+    # /usr/lib/python3.5/...
+    # TODO The implementation below is probably incorrect and not complete.
+    if 'dist-packages' in path or 'site-packages' in path:
+        return False
+
+    base_path = os.path.join(sys.prefix, 'lib', 'python')
+    return bool(re.match(re.escape(base_path) + r'\d.\d', path))
 
 
 def deep_ast_copy(obj):
@@ -40,10 +57,17 @@ def evaluate_call_of_leaf(context, leaf, cut_own_trailer=False):
 
     If you're using the leaf, e.g. the bracket `)` it will return ``list([])``.
 
-    # TODO remove cut_own_trailer option, since its always used with it. Just
-    #      ignore it, It's not what we want anyway. Or document it better?
+    We use this function for two purposes. Given an expression ``bar.foo``,
+    we may want to
+      - infer the type of ``foo`` to offer completions after foo
+      - infer the type of ``bar`` to be able to jump to the definition of foo
+    The option ``cut_own_trailer`` must be set to true for the second purpose.
     """
     trailer = leaf.parent
+    if trailer.type == 'fstring':
+        from jedi.evaluate import compiled
+        return compiled.get_string_context_set(context.evaluator)
+
     # The leaf may not be the last or first child, because there exist three
     # different trailers: `( x )`, `[ x ]` and `.x`. In the first two examples
     # we should not match anything more than x.
@@ -71,9 +95,14 @@ def evaluate_call_of_leaf(context, leaf, cut_own_trailer=False):
         base = power.children[0]
         trailers = power.children[1:cut]
 
+    if base == 'await':
+        base = trailers[0]
+        trailers = trailers[1:]
+
     values = context.eval_node(base)
+    from jedi.evaluate.syntax_tree import eval_trailer
     for trailer in trailers:
-        values = context.eval_trailer(values, trailer)
+        values = eval_trailer(context, values, trailer)
     return values
 
 
@@ -137,43 +166,104 @@ def get_module_names(module, all_scopes):
     Returns a dictionary with name parts as keys and their call paths as
     values.
     """
-    names = chain.from_iterable(module.used_names.values())
+    names = list(chain.from_iterable(module.get_used_names().values()))
     if not all_scopes:
         # We have to filter all the names that don't have the module as a
         # parent_scope. There's None as a parent, because nodes in the module
         # node have the parent module and not suite as all the others.
         # Therefore it's important to catch that case.
-        names = [n for n in names if n.get_parent_scope().parent in (module, None)]
+
+        def is_module_scope_name(name):
+            parent_scope = get_parent_scope(name)
+            # async functions have an extra wrapper. Strip it.
+            if parent_scope and parent_scope.type == 'async_stmt':
+                parent_scope = parent_scope.parent
+            return parent_scope in (module, None)
+
+        names = [n for n in names if is_module_scope_name(n)]
     return names
 
 
-class FakeName(tree.Name):
-    def __init__(self, name_str, parent=None, start_pos=(0, 0), is_definition=None):
-        """
-        In case is_definition is defined (not None), that bool value will be
-        returned.
-        """
-        super(FakeName, self).__init__(name_str, start_pos)
-        self.parent = parent
-        self._is_definition = is_definition
-
-    def get_definition(self):
-        return self.parent
-
-    def is_definition(self):
-        if self._is_definition is None:
-            return super(FakeName, self).is_definition()
-        else:
-            return self._is_definition
-
-
 @contextmanager
 def predefine_names(context, flow_scope, dct):
     predefined = context.predefined_names
-    if flow_scope in predefined:
-        raise NotImplementedError('Why does this happen?')
     predefined[flow_scope] = dct
     try:
         yield
     finally:
         del predefined[flow_scope]
+
+
+def is_string(context):
+    if context.evaluator.environment.version_info.major == 2:
+        str_classes = (unicode, bytes)
+    else:
+        str_classes = (unicode,)
+    return context.is_compiled() and isinstance(context.get_safe_value(default=None), str_classes)
+
+
+def is_literal(context):
+    return is_number(context) or is_string(context)
+
+
+def _get_safe_value_or_none(context, accept):
+    value = context.get_safe_value(default=None)
+    if isinstance(value, accept):
+        return value
+
+
+def get_int_or_none(context):
+    return _get_safe_value_or_none(context, int)
+
+
+def get_str_or_none(context):
+    return _get_safe_value_or_none(context, (bytes, unicode))
+
+
+def is_number(context):
+    return _get_safe_value_or_none(context, (int, float)) is not None
+
+
+class SimpleGetItemNotFound(Exception):
+    pass
+
+
+@contextmanager
+def reraise_getitem_errors(*exception_classes):
+    try:
+        yield
+    except exception_classes as e:
+        raise SimpleGetItemNotFound(e)
+
+
+def parse_dotted_names(nodes, is_import_from, until_node=None):
+    level = 0
+    names = []
+    for node in nodes[1:]:
+        if node in ('.', '...'):
+            if not names:
+                level += len(node.value)
+        elif node.type == 'dotted_name':
+            for n in node.children[::2]:
+                names.append(n)
+                if n is until_node:
+                    break
+            else:
+                continue
+            break
+        elif node.type == 'name':
+            names.append(node)
+            if node is until_node:
+                break
+        elif node == ',':
+            if not is_import_from:
+                names = []
+        else:
+            # Here if the keyword `import` comes along it stops checking
+            # for names.
+            break
+    return level, names
+
+
+def contexts_from_qualified_names(evaluator, *names):
+    return evaluator.import_module(names[:-1]).py__getattribute__(names[-1])

jedi/evaluate/imports.py

@@ -9,35 +9,58 @@ This module uses imp for python up to 3.2 and importlib for python 3.3 on; the
 correct implementation is delegated to _compatibility.
 
 This module also supports import autocompletion, which means to complete
-statements like ``from datetim`` (curser at the end would return ``datetime``).
+statements like ``from datetim`` (cursor at the end would return ``datetime``).
 """
-import imp
 import os
-import pkgutil
-import sys
 
-from jedi._compatibility import find_module, unicode
+from parso.python import tree
+from parso.tree import search_ancestor
+from parso import python_bytes_to_unicode
+
+from jedi._compatibility import (FileNotFoundError, ImplicitNSInfo,
+                                 force_unicode, unicode)
 from jedi import debug
 from jedi import settings
-from jedi.common import source_to_unicode, unite
-from jedi.parser.diff import FastParser
-from jedi.parser import tree
-from jedi.parser.utils import save_parser, load_parser, parser_cache
+from jedi.file_io import KnownContentFileIO, FileIO
+from jedi.parser_utils import get_cached_code_lines
 from jedi.evaluate import sys_path
 from jedi.evaluate import helpers
 from jedi.evaluate import compiled
 from jedi.evaluate import analysis
-from jedi.evaluate.cache import memoize_default
-from jedi.evaluate.filters import AbstractNameDefinition
+from jedi.evaluate.utils import unite
+from jedi.evaluate.cache import evaluator_method_cache
+from jedi.evaluate.names import ImportName, SubModuleName
+from jedi.evaluate.base_context import ContextSet, NO_CONTEXTS
+from jedi.evaluate.gradual.typeshed import import_module_decorator
+from jedi.evaluate.context.module import iter_module_names
+from jedi.plugins import plugin_manager
+
+
+class ModuleCache(object):
+    def __init__(self):
+        self._path_cache = {}
+        self._name_cache = {}
+
+    def add(self, string_names, context_set):
+        #path = module.py__file__()
+        #self._path_cache[path] = context_set
+        if string_names is not None:
+            self._name_cache[string_names] = context_set
+
+    def get(self, string_names):
+        return self._name_cache[string_names]
+
+    def get_from_path(self, path):
+        return self._path_cache[path]
 
 
 # This memoization is needed, because otherwise we will infinitely loop on
 # certain imports.
-@memoize_default(default=set())
+@evaluator_method_cache(default=NO_CONTEXTS)
 def infer_import(context, tree_name, is_goto=False):
     module_context = context.get_root_context()
-    import_node = tree_name.get_parent_until(tree.Import)
-    import_path = import_node.path_for_name(tree_name)
+    import_node = search_ancestor(tree_name, 'import_name', 'import_from')
+    import_path = import_node.get_path_for_name(tree_name)
     from_import_name = None
     evaluator = context.evaluator
     try:
@@ -60,14 +83,21 @@ def infer_import(context, tree_name, is_goto=False):
     #if import_node.is_nested() and not self.nested_resolve:
     #    scopes = [NestedImportModule(module, import_node)]
 
+    if not types:
+        return NO_CONTEXTS
+
     if from_import_name is not None:
         types = unite(
             t.py__getattribute__(
-                unicode(from_import_name),
+                from_import_name,
                 name_context=context,
-                is_goto=is_goto
-            ) for t in types
+                is_goto=is_goto,
+                analysis_errors=False
+            )
+            for t in types
         )
+        if not is_goto:
+            types = ContextSet(types)
 
         if not types:
             path = import_path + [from_import_name]
@@ -118,55 +148,38 @@ class NestedImportModule(tree.Module):
                                    self._nested_import)
 
 
-def _add_error(context, name, message=None):
-    # Should be a name, not a string!
-    if hasattr(name, 'parent'):
+def _add_error(context, name, message):
+    if hasattr(name, 'parent') and context is not None:
         analysis.add(context, 'import-error', name, message)
+    else:
+        debug.warning('ImportError without origin: ' + message)
 
 
-def get_init_path(directory_path):
+def _level_to_base_import_path(project_path, directory, level):
     """
-    The __init__ file can be searched in a directory. If found return it, else
-    None.
+    In case the level is outside of the currently known package (something like
+    import .....foo), we can still try our best to help the user for
+    completions.
     """
-    for suffix, _, _ in imp.get_suffixes():
-        path = os.path.join(directory_path, '__init__' + suffix)
-        if os.path.exists(path):
-            return path
-    return None
+    for i in range(level - 1):
+        old = directory
+        directory = os.path.dirname(directory)
+        if old == directory:
+            return None, None
 
-
-class ImportName(AbstractNameDefinition):
-    start_pos = (1, 0)
-
-    def __init__(self, parent_context, string_name):
-        self.parent_context = parent_context
-        self.string_name = string_name
-
-    def infer(self):
-        return Importer(
-            self.parent_context.evaluator,
-            [self.string_name],
-            self.parent_context,
-        ).follow()
-
-    def get_root_context(self):
-        # Not sure if this is correct.
-        return self.parent_context.get_root_context()
-
-    @property
-    def api_type(self):
-        return 'module'
-
-
-class SubModuleName(ImportName):
-    def infer(self):
-        return Importer(
-            self.parent_context.evaluator,
-            [self.string_name],
-            self.parent_context,
-            level=1
-        ).follow()
+    d = directory
+    level_import_paths = []
+    # Now that we are on the level that the user wants to be, calculate the
+    # import path for it.
+    while True:
+        if d == project_path:
+            return level_import_paths, d
+        dir_name = os.path.basename(d)
+        if dir_name:
+            level_import_paths.insert(0, dir_name)
+            d = os.path.dirname(d)
+        else:
+            return None, directory
 
 
 class Importer(object):
@@ -183,200 +196,132 @@ class Importer(object):
 
         :param import_path: List of namespaces (strings or Names).
         """
-        debug.speed('import %s' % (import_path,))
+        debug.speed('import %s %s' % (import_path, module_context))
         self._evaluator = evaluator
         self.level = level
         self.module_context = module_context
-        try:
-            self.file_path = module_context.py__file__()
-        except AttributeError:
-            # Can be None for certain compiled modules like 'builtins'.
-            self.file_path = None
 
+        self._fixed_sys_path = None
+        self._inference_possible = True
         if level:
-            base = module_context.py__package__().split('.')
-            if base == ['']:
-                base = []
-            if level > len(base):
-                path = module_context.py__file__()
-                if path is not None:
-                    import_path = list(import_path)
-                    for i in range(level):
-                        path = os.path.dirname(path)
-                    dir_name = os.path.basename(path)
-                    # This is not the proper way to do relative imports. However, since
-                    # Jedi cannot be sure about the entry point, we just calculate an
-                    # absolute path here.
-                    if dir_name:
-                        import_path.insert(0, dir_name)
-                    else:
-                        _add_error(module_context, import_path[-1])
-                        import_path = []
-                        # TODO add import error.
-                        debug.warning('Attempted relative import beyond top-level package.')
-            else:
+            base = module_context.py__package__()
+            # We need to care for two cases, the first one is if it's a valid
+            # Python import. This import has a properly defined module name
+            # chain like `foo.bar.baz` and an import in baz is made for
+            # `..lala.` It can then resolve to `foo.bar.lala`.
+            # The else here is a heuristic for all other cases, if for example
+            # in `foo` you search for `...bar`, it's obviously out of scope.
+            # However since Jedi tries to just do it's best, we help the user
+            # here, because he might have specified something wrong in his
+            # project.
+            if level <= len(base):
                 # Here we basically rewrite the level to 0.
-                import_path = tuple(base) + tuple(import_path)
+                base = tuple(base)
+                if level > 1:
+                    base = base[:-level + 1]
+                import_path = base + tuple(import_path)
+            else:
+                path = module_context.py__file__()
+                import_path = list(import_path)
+                if path is None:
+                    # If no path is defined, our best guess is that the current
+                    # file is edited by a user on the current working
+                    # directory. We need to add an initial path, because it
+                    # will get removed as the name of the current file.
+                    directory = os.getcwd()
+                else:
+                    directory = os.path.dirname(path)
+
+                base_import_path, base_directory = _level_to_base_import_path(
+                    self._evaluator.project._path, directory, level,
+                )
+                if base_directory is None:
+                    # Everything is lost, the relative import does point
+                    # somewhere out of the filesystem.
+                    self._inference_possible = False
+                else:
+                    self._fixed_sys_path = [force_unicode(base_directory)]
+
+                if base_import_path is None:
+                    if import_path:
+                        _add_error(
+                            module_context, import_path[0],
+                            message='Attempted relative import beyond top-level package.'
+                        )
+                else:
+                    import_path = base_import_path + import_path
         self.import_path = import_path
 
     @property
-    def str_import_path(self):
+    def _str_import_path(self):
         """Returns the import path as pure strings instead of `Name`."""
-        return tuple(str(name) for name in self.import_path)
+        return tuple(
+            name.value if isinstance(name, tree.Name) else name
+            for name in self.import_path
+        )
 
-    def sys_path_with_modifications(self):
-        in_path = []
-        sys_path_mod = list(sys_path.sys_path_with_modifications(
-            self._evaluator,
-            self.module_context
-        ))
-        if self.file_path is not None:
-            # If you edit e.g. gunicorn, there will be imports like this:
-            # `from gunicorn import something`. But gunicorn is not in the
-            # sys.path. Therefore look if gunicorn is a parent directory, #56.
-            if self.import_path:  # TODO is this check really needed?
-                for path in sys_path.traverse_parents(self.file_path):
-                    if os.path.basename(path) == self.str_import_path[0]:
-                        in_path.append(os.path.dirname(path))
+    def _sys_path_with_modifications(self):
+        if self._fixed_sys_path is not None:
+            return self._fixed_sys_path
 
-            # Since we know nothing about the call location of the sys.path,
-            # it's a possibility that the current directory is the origin of
-            # the Python execution.
-            sys_path_mod.insert(0, os.path.dirname(self.file_path))
+        sys_path_mod = (
+            self._evaluator.get_sys_path()
+            + sys_path.check_sys_path_modifications(self.module_context)
+        )
 
-        return in_path + sys_path_mod
+        if self._evaluator.environment.version_info.major == 2:
+            file_path = self.module_context.py__file__()
+            if file_path is not None:
+                # Python2 uses an old strange way of importing relative imports.
+                sys_path_mod.append(force_unicode(os.path.dirname(file_path)))
+
+        return sys_path_mod
 
     def follow(self):
-        if not self.import_path:
-            return set()
-        return self._do_import(self.import_path, self.sys_path_with_modifications())
+        if not self.import_path or not self._inference_possible:
+            return NO_CONTEXTS
 
-    def _do_import(self, import_path, sys_path):
-        """
-        This method is very similar to importlib's `_gcd_import`.
-        """
-        import_parts = [str(i) for i in import_path]
+        import_names = tuple(
+            force_unicode(i.value if isinstance(i, tree.Name) else i)
+            for i in self.import_path
+        )
+        sys_path = self._sys_path_with_modifications()
 
-        # Handle "magic" Flask extension imports:
-        # ``flask.ext.foo`` is really ``flask_foo`` or ``flaskext.foo``.
-        if len(import_path) > 2 and import_parts[:2] == ['flask', 'ext']:
-            # New style.
-            ipath = ('flask_' + str(import_parts[2]),) + import_path[3:]
-            modules = self._do_import(ipath, sys_path)
-            if modules:
-                return modules
-            else:
-                # Old style
-                return self._do_import(('flaskext',) + import_path[2:], sys_path)
-
-        module_name = '.'.join(import_parts)
-        try:
-            return set([self._evaluator.modules[module_name]])
-        except KeyError:
-            pass
-
-        if len(import_path) > 1:
-            # This is a recursive way of importing that works great with
-            # the module cache.
-            bases = self._do_import(import_path[:-1], sys_path)
-            if not bases:
-                return set()
-            # We can take the first element, because only the os special
-            # case yields multiple modules, which is not important for
-            # further imports.
-            parent_module = list(bases)[0]
-
-            # This is a huge exception, we follow a nested import
-            # ``os.path``, because it's a very important one in Python
-            # that is being achieved by messing with ``sys.modules`` in
-            # ``os``.
-            if [str(i) for i in import_path] == ['os', 'path']:
-                return parent_module.py__getattribute__('path')
-
-            try:
-                method = parent_module.py__path__
-            except AttributeError:
-                # The module is not a package.
-                _add_error(parent_module, import_path[-1])
-                return set()
-            else:
-                paths = method()
-                debug.dbg('search_module %s in paths %s', module_name, paths)
-                for path in paths:
-                    # At the moment we are only using one path. So this is
-                    # not important to be correct.
-                    try:
-                        module_file, module_path, is_pkg = \
-                            find_module(import_parts[-1], [path])
-                        break
-                    except ImportError:
-                        module_path = None
-                if module_path is None:
-                    _add_error(parent_module, import_path[-1])
-                    return set()
-        else:
-            parent_module = None
-            try:
-                debug.dbg('search_module %s in %s', import_parts[-1], self.file_path)
-                # Override the sys.path. It works only good that way.
-                # Injecting the path directly into `find_module` did not work.
-                sys.path, temp = sys_path, sys.path
-                try:
-                    module_file, module_path, is_pkg = \
-                        find_module(import_parts[-1])
-                finally:
-                    sys.path = temp
-            except ImportError:
-                # The module is not a package.
-                _add_error(self.module_context, import_path[-1])
-                return set()
-
-        source = None
-        if is_pkg:
-            # In this case, we don't have a file yet. Search for the
-            # __init__ file.
-            if module_path.endswith(('.zip', '.egg')):
-                source = module_file.loader.get_source(module_name)
-            else:
-                module_path = get_init_path(module_path)
-        elif module_file:
-            source = module_file.read()
-            module_file.close()
-
-        if module_file is None and not module_path.endswith(('.py', '.zip', '.egg')):
-            module = compiled.load_module(self._evaluator, module_path)
-        else:
-            module = _load_module(self._evaluator, module_path, source, sys_path, parent_module)
-
-        if module is None:
-            # The file might raise an ImportError e.g. and therefore not be
-            # importable.
-            return set()
-
-        self._evaluator.modules[module_name] = module
-        return set([module])
-
-    def _generate_name(self, name, in_module=None):
-        # Create a pseudo import to be able to follow them.
-        if in_module is None:
-            return ImportName(self.module_context, name)
-        return SubModuleName(in_module, name)
+        context_set = [None]
+        for i, name in enumerate(self.import_path):
+            context_set = ContextSet.from_sets([
+                self._evaluator.import_module(
+                    import_names[:i+1],
+                    parent_module_context,
+                    sys_path
+                ) for parent_module_context in context_set
+            ])
+            if not context_set:
+                message = 'No module named ' + '.'.join(import_names)
+                _add_error(self.module_context, name, message)
+                return NO_CONTEXTS
+        return context_set
 
     def _get_module_names(self, search_path=None, in_module=None):
         """
         Get the names of all modules in the search_path. This means file names
         and not names defined in the files.
         """
-
         names = []
         # add builtin module names
         if search_path is None and in_module is None:
-            names += [self._generate_name(name) for name in sys.builtin_module_names]
+            names += [ImportName(self.module_context, name)
+                      for name in self._evaluator.compiled_subprocess.get_builtin_module_names()]
 
         if search_path is None:
-            search_path = self.sys_path_with_modifications()
-        for module_loader, name, is_pkg in pkgutil.iter_modules(search_path):
-            names.append(self._generate_name(name, in_module=in_module))
+            search_path = self._sys_path_with_modifications()
+
+        for name in iter_module_names(self._evaluator, search_path):
+            if in_module is None:
+                n = ImportName(self.module_context, name)
+            else:
+                n = SubModuleName(in_module, name)
+            names.append(n)
         return names
 
     def completion_names(self, evaluator, only_modules=False):
@@ -384,151 +329,240 @@ class Importer(object):
         :param only_modules: Indicates wheter it's possible to import a
             definition that is not defined in a module.
         """
-        from jedi.evaluate.representation import ModuleContext
+        if not self._inference_possible:
+            return []
+
         names = []
         if self.import_path:
             # flask
-            if self.str_import_path == ('flask', 'ext'):
+            if self._str_import_path == ('flask', 'ext'):
                 # List Flask extensions like ``flask_foo``
                 for mod in self._get_module_names():
                     modname = mod.string_name
                     if modname.startswith('flask_'):
                         extname = modname[len('flask_'):]
-                        names.append(self._generate_name(extname))
+                        names.append(ImportName(self.module_context, extname))
                 # Now the old style: ``flaskext.foo``
-                for dir in self.sys_path_with_modifications():
+                for dir in self._sys_path_with_modifications():
                     flaskext = os.path.join(dir, 'flaskext')
                     if os.path.isdir(flaskext):
                         names += self._get_module_names([flaskext])
 
-            for context in self.follow():
+            contexts = self.follow()
+            for context in contexts:
                 # Non-modules are not completable.
                 if context.api_type != 'module':  # not a module
                     continue
+                names += context.sub_modules_dict().values()
 
-                # namespace packages
-                if isinstance(context, ModuleContext) and \
-                        context.py__file__().endswith('__init__.py'):
-                    paths = context.py__path__()
-                    names += self._get_module_names(paths, in_module=context)
+            if not only_modules:
+                from jedi.evaluate.gradual.conversion import convert_contexts
 
-                if only_modules:
-                    # In the case of an import like `from x.` we don't need to
-                    # add all the variables.
-                    if ('os',) == self.str_import_path and not self.level:
-                        # os.path is a hardcoded exception, because it's a
-                        # ``sys.modules`` modification.
-                        names.append(self._generate_name('path'))
-
-                    continue
-
-                for filter in context.get_filters(search_global=False):
-                    names += filter.values()
+                both_contexts = contexts | convert_contexts(contexts)
+                for c in both_contexts:
+                    for filter in c.get_filters(search_global=False):
+                        names += filter.values()
         else:
-            # Empty import path=completion after import
-            if not self.level:
+            if self.level:
+                # We only get here if the level cannot be properly calculated.
+                names += self._get_module_names(self._fixed_sys_path)
+            else:
+                # This is just the list of global imports.
                 names += self._get_module_names()
-
-            if self.file_path is not None:
-                path = os.path.abspath(self.file_path)
-                for i in range(self.level - 1):
-                    path = os.path.dirname(path)
-                names += self._get_module_names([path])
-
         return names
 
 
-def _load_module(evaluator, path=None, source=None, sys_path=None, parent_module=None):
-    def load(source):
-        dotted_path = path and compiled.dotted_from_fs_path(path, sys_path)
-        if path is not None and path.endswith(('.py', '.zip', '.egg')) \
-                and dotted_path not in settings.auto_import_modules:
-            if source is None:
-                with open(path, 'rb') as f:
-                    source = f.read()
+@plugin_manager.decorate()
+@import_module_decorator
+def import_module(evaluator, import_names, parent_module_context, sys_path):
+    """
+    This method is very similar to importlib's `_gcd_import`.
+    """
+    if import_names[0] in settings.auto_import_modules:
+        module = _load_builtin_module(evaluator, import_names, sys_path)
+        if module is None:
+            return NO_CONTEXTS
+        return ContextSet([module])
+
+    module_name = '.'.join(import_names)
+    if parent_module_context is None:
+        # Override the sys.path. It works only good that way.
+        # Injecting the path directly into `find_module` did not work.
+        file_io_or_ns, is_pkg = evaluator.compiled_subprocess.get_module_info(
+            string=import_names[-1],
+            full_name=module_name,
+            sys_path=sys_path,
+            is_global_search=True,
+        )
+        if is_pkg is None:
+            return NO_CONTEXTS
+    else:
+        try:
+            method = parent_module_context.py__path__
+        except AttributeError:
+            # The module is not a package.
+            return NO_CONTEXTS
         else:
-            return compiled.load_module(evaluator, path)
-        p = path
-        p = FastParser(evaluator.grammar, source_to_unicode(source), p)
-        save_parser(path, p)
-        return p.module
+            paths = method()
+            for path in paths:
+                # At the moment we are only using one path. So this is
+                # not important to be correct.
+                if not isinstance(path, list):
+                    path = [path]
+                file_io_or_ns, is_pkg = evaluator.compiled_subprocess.get_module_info(
+                    string=import_names[-1],
+                    path=path,
+                    full_name=module_name,
+                    is_global_search=False,
+                )
+                if is_pkg is not None:
+                    break
+            else:
+                return NO_CONTEXTS
 
+    if isinstance(file_io_or_ns, ImplicitNSInfo):
+        from jedi.evaluate.context.namespace import ImplicitNamespaceContext
+        module = ImplicitNamespaceContext(
+            evaluator,
+            fullname=file_io_or_ns.name,
+            paths=file_io_or_ns.paths,
+        )
+    elif file_io_or_ns is None:
+        module = _load_builtin_module(evaluator, import_names, sys_path)
+        if module is None:
+            return NO_CONTEXTS
+    else:
+        module = _load_python_module(
+            evaluator, file_io_or_ns, sys_path,
+            import_names=import_names,
+            is_package=is_pkg,
+        )
+
+    if parent_module_context is None:
+        debug.dbg('global search_module %s: %s', import_names[-1], module)
+    else:
+        debug.dbg('search_module %s in paths %s: %s', module_name, paths, module)
+    return ContextSet([module])
+
+
+def _load_python_module(evaluator, file_io, sys_path=None,
+                        import_names=None, is_package=False):
+    try:
+        return evaluator.module_cache.get_from_path(file_io.path)
+    except KeyError:
+        pass
+
+    module_node = evaluator.parse(
+        file_io=file_io,
+        cache=True,
+        diff_cache=settings.fast_parser,
+        cache_path=settings.cache_directory
+    )
+
+    from jedi.evaluate.context import ModuleContext
+    return ModuleContext(
+        evaluator, module_node,
+        file_io=file_io,
+        string_names=import_names,
+        code_lines=get_cached_code_lines(evaluator.grammar, file_io.path),
+        is_package=is_package,
+    )
+
+
+def _load_builtin_module(evaluator, import_names=None, sys_path=None):
     if sys_path is None:
-        sys_path = evaluator.sys_path
+        sys_path = evaluator.get_sys_path()
 
-    cached = load_parser(path)
-    module_node = load(source) if cached is None else cached.module
-    if isinstance(module_node, compiled.CompiledObject):
-        return module_node
-
-    from jedi.evaluate.representation import ModuleContext
-    return ModuleContext(evaluator, module_node)
+    dotted_name = '.'.join(import_names)
+    assert dotted_name is not None
+    module = compiled.load_module(evaluator, dotted_name=dotted_name, sys_path=sys_path)
+    if module is None:
+        # The file might raise an ImportError e.g. and therefore not be
+        # importable.
+        return None
+    return module
 
 
-def add_module(evaluator, module_name, module):
-    if '.' not in module_name:
-        # We cannot add paths with dots, because that would collide with
-        # the sepatator dots for nested packages. Therefore we return
-        # `__main__` in ModuleWrapper.py__name__(), which is similar to
-        # Python behavior.
-        evaluator.modules[module_name] = module
+def _load_module_from_path(evaluator, file_io, base_names):
+    """
+    This should pretty much only be used for get_modules_containing_name. It's
+    here to ensure that a random path is still properly loaded into the Jedi
+    module structure.
+    """
+    e_sys_path = evaluator.get_sys_path()
+    path = file_io.path
+    if base_names:
+        module_name = os.path.basename(path)
+        module_name = sys_path.remove_python_path_suffix(module_name)
+        is_package = module_name == '__init__'
+        if is_package:
+            import_names = base_names
+        else:
+            import_names = base_names + (module_name,)
+    else:
+        import_names, is_package = sys_path.transform_path_to_dotted(e_sys_path, path)
+
+    module = _load_python_module(
+        evaluator, file_io,
+        sys_path=e_sys_path,
+        import_names=import_names,
+        is_package=is_package,
+    )
+    evaluator.module_cache.add(import_names, ContextSet([module]))
+    return module
 
 
 def get_modules_containing_name(evaluator, modules, name):
     """
     Search a name in the directories of modules.
     """
-    from jedi.evaluate import representation as er
+    def check_directory(folder_io):
+        for file_name in folder_io.list():
+            if file_name.endswith('.py'):
+                yield folder_io.get_file_io(file_name)
 
-    def check_python_file(path):
+    def check_fs(file_io, base_names):
         try:
-            parser_cache_item = parser_cache[path]
-        except KeyError:
-            try:
-                return check_fs(path)
-            except IOError:
-                return None
-        else:
-            return er.ModuleContext(evaluator, parser_cache_item.parser.module)
-
-    def check_fs(path):
-        with open(path, 'rb') as f:
-            source = source_to_unicode(f.read())
-            if name in source:
-                module_name = os.path.basename(path)[:-3]  # Remove `.py`.
-                module = _load_module(evaluator, path, source)
-                add_module(evaluator, module_name, module)
-                return module
+            code = file_io.read()
+        except FileNotFoundError:
+            return None
+        code = python_bytes_to_unicode(code, errors='replace')
+        if name not in code:
+            return None
+        new_file_io = KnownContentFileIO(file_io.path, code)
+        m = _load_module_from_path(evaluator, new_file_io, base_names)
+        if isinstance(m, compiled.CompiledObject):
+            return None
+        return m
 
     # skip non python modules
     used_mod_paths = set()
+    folders_with_names_to_be_checked = []
     for m in modules:
-        try:
-            path = m.py__file__()
-        except AttributeError:
-            pass
-        else:
-            used_mod_paths.add(path)
+        if m.file_io is not None:
+            path = m.file_io.path
+            if path not in used_mod_paths:
+                used_mod_paths.add(path)
+                folders_with_names_to_be_checked.append((
+                    m.file_io.get_parent_folder(),
+                    m.py__package__()
+                ))
         yield m
 
     if not settings.dynamic_params_for_other_modules:
         return
 
-    paths = set(settings.additional_dynamic_modules)
-    for p in used_mod_paths:
-        if p is not None:
-            # We need abspath, because the seetings paths might not already
-            # have been converted to absolute paths.
-            d = os.path.dirname(os.path.abspath(p))
-            for file_name in os.listdir(d):
-                path = os.path.join(d, file_name)
-                if path not in used_mod_paths and path not in paths:
-                    if file_name.endswith('.py'):
-                        paths.add(path)
+    def get_file_ios_to_check():
+        for folder_io, base_names in folders_with_names_to_be_checked:
+            for file_io in check_directory(folder_io):
+                yield file_io, base_names
 
-    # Sort here to make issues less random.
-    for p in sorted(paths):
-        # make testing easier, sort it - same results on every interpreter
-        m = check_python_file(p)
-        if m is not None and not isinstance(m, compiled.CompiledObject):
+        for p in settings.additional_dynamic_modules:
+            p = os.path.abspath(p)
+            if p not in used_mod_paths:
+                yield FileIO(p), None
+
+    for file_io, base_names in get_file_ios_to_check():
+        m = check_fs(file_io, base_names)
+        if m is not None:
             yield m

jedi/evaluate/instance.py

@@ -1,445 +0,0 @@
-from abc import abstractproperty
-
-from jedi._compatibility import is_py3
-from jedi.common import unite
-from jedi import debug
-from jedi.evaluate import compiled
-from jedi.evaluate import filters
-from jedi.evaluate.context import Context, LazyKnownContext, LazyKnownContexts
-from jedi.evaluate.cache import memoize_default
-from jedi.cache import memoize_method
-from jedi.evaluate import representation as er
-from jedi.evaluate.dynamic import search_params
-from jedi.evaluate import iterable
-
-
-class AbstractInstanceContext(Context):
-    """
-    This class is used to evaluate instances.
-    """
-    api_type = 'instance'
-
-    def __init__(self, evaluator, parent_context, class_context, var_args):
-        super(AbstractInstanceContext, self).__init__(evaluator, parent_context)
-        # Generated instances are classes that are just generated by self
-        # (No var_args) used.
-        self.class_context = class_context
-        self.var_args = var_args
-
-    def is_class(self):
-        return False
-
-    @property
-    def py__call__(self):
-        names = self.get_function_slot_names('__call__')
-        if not names:
-            # Means the Instance is not callable.
-            raise AttributeError
-
-        def execute(arguments):
-            return unite(name.execute(arguments) for name in names)
-
-        return execute
-
-    def py__class__(self):
-        return self.class_context
-
-    def py__bool__(self):
-        # Signalize that we don't know about the bool type.
-        return None
-
-    def get_function_slot_names(self, name):
-        # Python classes don't look at the dictionary of the instance when
-        # looking up `__call__`. This is something that has to do with Python's
-        # internal slot system (note: not __slots__, but C slots).
-        for filter in self.get_filters(include_self_names=False):
-            names = filter.get(name)
-            if names:
-                return names
-        return []
-
-    def execute_function_slots(self, names, *evaluated_args):
-        return unite(
-            name.execute_evaluated(*evaluated_args)
-            for name in names
-        )
-
-    def py__get__(self, obj):
-        # Arguments in __get__ descriptors are obj, class.
-        # `method` is the new parent of the array, don't know if that's good.
-        names = self.get_function_slot_names('__get__')
-        if names:
-            if isinstance(obj, AbstractInstanceContext):
-                return self.execute_function_slots(names, obj, obj.class_context)
-            else:
-                none_obj = compiled.create(self.evaluator, None)
-                return self.execute_function_slots(names, none_obj, obj)
-        else:
-            return set([self])
-
-    def get_filters(self, search_global=None, until_position=None,
-                    origin_scope=None, include_self_names=True):
-        if include_self_names:
-            for cls in self.class_context.py__mro__():
-                if isinstance(cls, compiled.CompiledObject):
-                    if cls.tree_node is not None:
-                        # In this case we're talking about a fake object, it
-                        # doesn't make sense for normal compiled objects to
-                        # search for self variables.
-                        yield SelfNameFilter(self.evaluator, self, cls, origin_scope)
-                else:
-                    yield SelfNameFilter(self.evaluator, self, cls, origin_scope)
-
-        for cls in self.class_context.py__mro__():
-            if isinstance(cls, compiled.CompiledObject):
-                yield CompiledInstanceClassFilter(self.evaluator, self, cls)
-            else:
-                yield InstanceClassFilter(self.evaluator, self, cls, origin_scope)
-
-    def py__getitem__(self, index):
-        try:
-            names = self.get_function_slot_names('__getitem__')
-        except KeyError:
-            debug.warning('No __getitem__, cannot access the array.')
-            return set()
-        else:
-            index_obj = compiled.create(self.evaluator, index)
-            return self.execute_function_slots(names, index_obj)
-
-    def py__iter__(self):
-        iter_slot_names = self.get_function_slot_names('__iter__')
-        if not iter_slot_names:
-            debug.warning('No __iter__ on %s.' % self)
-            return
-
-        for generator in self.execute_function_slots(iter_slot_names):
-            if isinstance(generator, AbstractInstanceContext):
-                # `__next__` logic.
-                name = '__next__' if is_py3 else 'next'
-                iter_slot_names = generator.get_function_slot_names(name)
-                if iter_slot_names:
-                    yield LazyKnownContexts(
-                        generator.execute_function_slots(iter_slot_names)
-                    )
-                else:
-                    debug.warning('Instance has no __next__ function in %s.', generator)
-            else:
-                for lazy_context in generator.py__iter__():
-                    yield lazy_context
-
-    @abstractproperty
-    def name(self):
-        pass
-
-    def _create_init_execution(self, class_context, func_node):
-        bound_method = BoundMethod(
-            self.evaluator, self, class_context, self.parent_context, func_node
-        )
-        return InstanceFunctionExecution(
-            self,
-            class_context.parent_context,
-            bound_method,
-            self.var_args
-        )
-
-    def create_init_executions(self):
-        for name in self.get_function_slot_names('__init__'):
-            if isinstance(name, LazyInstanceName):
-                yield self._create_init_execution(name.class_context, name.tree_name.parent)
-
-    @memoize_default()
-    def create_instance_context(self, class_context, node):
-        if node.parent.type in ('funcdef', 'classdef'):
-            node = node.parent
-        scope = node.get_parent_scope()
-        if scope == class_context.tree_node:
-            return class_context
-        else:
-            parent_context = self.create_instance_context(class_context, scope)
-            if scope.type == 'funcdef':
-                if scope.name.value == '__init__' and parent_context == class_context:
-                    return self._create_init_execution(class_context, scope)
-                else:
-                    bound_method = BoundMethod(
-                        self.evaluator, self, class_context,
-                        self.parent_context, scope
-                    )
-                    return bound_method.get_function_execution()
-            else:
-                raise NotImplementedError
-        return class_context
-
-    def __repr__(self):
-        return "<%s of %s(%s)>" % (self.__class__.__name__, self.class_context,
-                                   self.var_args)
-
-
-class CompiledInstance(AbstractInstanceContext):
-    def __init__(self, *args, **kwargs):
-        super(CompiledInstance, self).__init__(*args, **kwargs)
-        # I don't think that dynamic append lookups should happen here. That
-        # sounds more like something that should go to py__iter__.
-        if self.class_context.name.string_name in ['list', 'set'] \
-                and self.parent_context.get_root_context() == self.evaluator.BUILTINS:
-            # compare the module path with the builtin name.
-            self.var_args = iterable.get_dynamic_array_instance(self)
-
-    @property
-    def name(self):
-        return compiled.CompiledContextName(self, self.class_context.name.string_name)
-
-    def create_instance_context(self, class_context, node):
-        if node.get_parent_scope().type == 'classdef':
-            return class_context
-        else:
-            return super(CompiledInstance, self).create_instance_context(class_context, node)
-
-
-class TreeInstance(AbstractInstanceContext):
-    @property
-    def name(self):
-        return filters.ContextName(self, self.class_context.name.tree_name)
-
-
-class AnonymousInstance(TreeInstance):
-    def __init__(self, evaluator, parent_context, class_context):
-        super(AnonymousInstance, self).__init__(
-            evaluator,
-            parent_context,
-            class_context,
-            var_args=None
-        )
-
-
-class CompiledInstanceName(compiled.CompiledName):
-    def __init__(self, evaluator, instance, parent_context, name):
-        super(CompiledInstanceName, self).__init__(evaluator, parent_context, name)
-        self._instance = instance
-
-    def infer(self):
-        for result_context in super(CompiledInstanceName, self).infer():
-            if isinstance(result_context, er.FunctionContext):
-                parent_context = result_context.parent_context
-                while parent_context.is_class():
-                    parent_context = parent_context.parent_context
-
-                yield BoundMethod(
-                    result_context.evaluator, self._instance, self.parent_context,
-                    parent_context, result_context.tree_node
-                )
-            else:
-                if result_context.api_type == 'function':
-                    yield CompiledBoundMethod(result_context)
-                else:
-                    yield result_context
-
-
-class CompiledInstanceClassFilter(compiled.CompiledObjectFilter):
-    name_class = CompiledInstanceName
-
-    def __init__(self, evaluator, instance, compiled_object):
-        super(CompiledInstanceClassFilter, self).__init__(
-            evaluator,
-            compiled_object,
-            is_instance=True,
-        )
-        self._instance = instance
-
-    def _create_name(self, name):
-        return self.name_class(
-            self._evaluator, self._instance, self._compiled_object, name)
-
-
-class BoundMethod(er.FunctionContext):
-    def __init__(self, evaluator, instance, class_context, *args, **kwargs):
-        super(BoundMethod, self).__init__(evaluator, *args, **kwargs)
-        self._instance = instance
-        self._class_context = class_context
-
-    def get_function_execution(self, arguments=None):
-        if arguments is None:
-            return AnonymousInstanceFunctionExecution(
-                self._instance, self.parent_context, self)
-        else:
-            return InstanceFunctionExecution(
-                self._instance, self.parent_context, self, arguments)
-
-
-class CompiledBoundMethod(compiled.CompiledObject):
-    def __init__(self, func):
-        super(CompiledBoundMethod, self).__init__(
-            func.evaluator, func.obj, func.parent_context, func.tree_node)
-
-    def get_param_names(self):
-        return list(super(CompiledBoundMethod, self).get_param_names())[1:]
-
-
-class InstanceNameDefinition(filters.TreeNameDefinition):
-    def infer(self):
-        contexts = super(InstanceNameDefinition, self).infer()
-        for context in contexts:
-            yield context
-
-
-class LazyInstanceName(filters.TreeNameDefinition):
-    """
-    This name calculates the parent_context lazily.
-    """
-    def __init__(self, instance, class_context, tree_name):
-        self._instance = instance
-        self.class_context = class_context
-        self.tree_name = tree_name
-
-    @property
-    def parent_context(self):
-        return self._instance.create_instance_context(self.class_context, self.tree_name)
-
-
-class LazyInstanceClassName(LazyInstanceName):
-    def infer(self):
-        for result_context in super(LazyInstanceClassName, self).infer():
-            if isinstance(result_context, er.FunctionContext):
-                # Classes are never used to resolve anything within the
-                # functions. Only other functions and modules will resolve
-                # those things.
-                parent_context = result_context.parent_context
-                while parent_context.is_class():
-                    parent_context = parent_context.parent_context
-
-                yield BoundMethod(
-                    result_context.evaluator, self._instance, self.class_context,
-                    parent_context, result_context.tree_node
-                )
-            else:
-                for c in er.apply_py__get__(result_context, self._instance):
-                    yield c
-
-
-class InstanceClassFilter(filters.ParserTreeFilter):
-    name_class = LazyInstanceClassName
-
-    def __init__(self, evaluator, context, class_context, origin_scope):
-        super(InstanceClassFilter, self).__init__(
-            evaluator=evaluator,
-            context=context,
-            node_context=class_context,
-            origin_scope=origin_scope
-        )
-        self._class_context = class_context
-
-    def _equals_origin_scope(self):
-        node = self._origin_scope
-        while node is not None:
-            if node == self._parser_scope or node == self.context:
-                return True
-            node = node.get_parent_scope()
-        return False
-
-    def _access_possible(self, name):
-        return not name.value.startswith('__') or name.value.endswith('__') \
-            or self._equals_origin_scope()
-
-    def _filter(self, names):
-        names = super(InstanceClassFilter, self)._filter(names)
-        return [name for name in names if self._access_possible(name)]
-
-    def _convert_names(self, names):
-        return [self.name_class(self.context, self._class_context, name) for name in names]
-
-
-class SelfNameFilter(InstanceClassFilter):
-    name_class = LazyInstanceName
-
-    def _filter(self, names):
-        names = self._filter_self_names(names)
-        if isinstance(self._parser_scope, compiled.CompiledObject) and False:
-            # This would be for builtin skeletons, which are not yet supported.
-            return list(names)
-        else:
-            start, end = self._parser_scope.start_pos, self._parser_scope.end_pos
-            return [n for n in names if start < n.start_pos < end]
-
-    def _filter_self_names(self, names):
-        for name in names:
-            trailer = name.parent
-            if trailer.type == 'trailer' \
-                    and len(trailer.children) == 2 \
-                    and trailer.children[0] == '.':
-                if name.is_definition() and self._access_possible(name):
-                    yield name
-
-    def _check_flows(self, names):
-        return names
-
-
-class ParamArguments(object):
-    """
-    TODO This seems like a strange class, clean up?
-    """
-    class LazyParamContext(object):
-        def __init__(self, fucking_param):
-            self._param = fucking_param
-
-        def infer(self):
-            return self._param.infer()
-
-    def __init__(self, class_context, funcdef):
-        self._class_context = class_context
-        self._funcdef = funcdef
-
-    def unpack(self, func=None):
-        params = search_params(
-            self._class_context.evaluator,
-            self._class_context,
-            self._funcdef
-        )
-        is_first = True
-        for p in params:
-            # TODO Yeah, here at last, the class seems to be really wrong.
-            if is_first:
-                is_first = False
-                continue
-            yield None, self.LazyParamContext(p)
-
-
-class InstanceVarArgs(object):
-    def __init__(self, instance, funcdef, var_args):
-        self._instance = instance
-        self._funcdef = funcdef
-        self._var_args = var_args
-
-    @memoize_method
-    def _get_var_args(self):
-        if self._var_args is None:
-            # TODO this parent_context might be wrong. test?!
-            return ParamArguments(self._instance.class_context, self._funcdef)
-
-        return self._var_args
-
-    def unpack(self, func=None):
-        yield None, LazyKnownContext(self._instance)
-        for values in self._get_var_args().unpack(func):
-            yield values
-
-    def get_calling_nodes(self):
-        return self._get_var_args().get_calling_nodes()
-
-    def __getattr__(self, name):
-        return getattr(self._var_args, name)
-
-
-class InstanceFunctionExecution(er.FunctionExecutionContext):
-    def __init__(self, instance, parent_context, function_context, var_args):
-        self.instance = instance
-        var_args = InstanceVarArgs(instance, function_context.tree_node, var_args)
-
-        super(InstanceFunctionExecution, self).__init__(
-            instance.evaluator, parent_context, function_context, var_args)
-
-
-class AnonymousInstanceFunctionExecution(InstanceFunctionExecution):
-    function_execution_filter = filters.AnonymousInstanceFunctionExecutionFilter
-
-    def __init__(self, instance, parent_context, function_context):
-        super(AnonymousInstanceFunctionExecution, self).__init__(
-            instance, parent_context, function_context, None)

jedi/evaluate/iterable.py

@@ -1,892 +0,0 @@
-"""
-Contains all classes and functions to deal with lists, dicts, generators and
-iterators in general.
-
-Array modifications
-*******************
-
-If the content of an array (``set``/``list``) is requested somewhere, the
-current module will be checked for appearances of ``arr.append``,
-``arr.insert``, etc.  If the ``arr`` name points to an actual array, the
-content will be added
-
-This can be really cpu intensive, as you can imagine. Because |jedi| has to
-follow **every** ``append`` and check wheter it's the right array. However this
-works pretty good, because in *slow* cases, the recursion detector and other
-settings will stop this process.
-
-It is important to note that:
-
-1. Array modfications work only in the current module.
-2. Jedi only checks Array additions; ``list.pop``, etc are ignored.
-"""
-from jedi import debug
-from jedi import settings
-from jedi import common
-from jedi.common import unite, safe_property
-from jedi._compatibility import unicode, zip_longest, is_py3
-from jedi.evaluate import compiled
-from jedi.evaluate import helpers
-from jedi.evaluate import analysis
-from jedi.evaluate import pep0484
-from jedi.evaluate import context
-from jedi.evaluate import precedence
-from jedi.evaluate import recursion
-from jedi.evaluate.cache import memoize_default
-from jedi.evaluate.filters import DictFilter, AbstractNameDefinition, \
-    ParserTreeFilter
-
-
-class AbstractSequence(context.Context):
-    builtin_methods = {}
-    api_type = 'instance'
-
-    def __init__(self, evaluator):
-        super(AbstractSequence, self).__init__(evaluator, evaluator.BUILTINS)
-
-    def get_filters(self, search_global, until_position=None, origin_scope=None):
-        raise NotImplementedError
-
-    @property
-    def name(self):
-        return compiled.CompiledContextName(self, self.array_type)
-
-
-class BuiltinMethod(object):
-    """``Generator.__next__`` ``dict.values`` methods and so on."""
-    def __init__(self, builtin_context, method, builtin_func):
-        self._builtin_context = builtin_context
-        self._method = method
-        self._builtin_func = builtin_func
-
-    def py__call__(self, params):
-        return self._method(self._builtin_context)
-
-    def __getattr__(self, name):
-        return getattr(self._builtin_func, name)
-
-
-class SpecialMethodFilter(DictFilter):
-    """
-    A filter for methods that are defined in this module on the corresponding
-    classes like Generator (for __next__, etc).
-    """
-    class SpecialMethodName(AbstractNameDefinition):
-        api_type = 'function'
-
-        def __init__(self, parent_context, string_name, callable_, builtin_context):
-            self.parent_context = parent_context
-            self.string_name = string_name
-            self._callable = callable_
-            self._builtin_context = builtin_context
-
-        def infer(self):
-            filter = next(self._builtin_context.get_filters())
-            # We can take the first index, because on builtin methods there's
-            # always only going to be one name. The same is true for the
-            # inferred values.
-            builtin_func = next(iter(filter.get(self.string_name)[0].infer()))
-            return set([BuiltinMethod(self.parent_context, self._callable, builtin_func)])
-
-    def __init__(self, context, dct, builtin_context):
-        super(SpecialMethodFilter, self).__init__(dct)
-        self.context = context
-        self._builtin_context = builtin_context
-        """
-        This context is what will be used to introspect the name, where as the
-        other context will be used to execute the function.
-
-        We distinguish, because we have to.
-        """
-
-    def _convert(self, name, value):
-        return self.SpecialMethodName(self.context, name, value, self._builtin_context)
-
-
-def has_builtin_methods(cls):
-    base_dct = {}
-    # Need to care properly about inheritance. Builtin Methods should not get
-    # lost, just because they are not mentioned in a class.
-    for base_cls in reversed(cls.__bases__):
-        try:
-            base_dct.update(base_cls.builtin_methods)
-        except AttributeError:
-            pass
-
-    cls.builtin_methods = base_dct
-    for func in cls.__dict__.values():
-        try:
-            cls.builtin_methods.update(func.registered_builtin_methods)
-        except AttributeError:
-            pass
-    return cls
-
-
-def register_builtin_method(method_name, python_version_match=None):
-    def wrapper(func):
-        if python_version_match and python_version_match != 2 + int(is_py3):
-            # Some functions do only apply to certain versions.
-            return func
-        dct = func.__dict__.setdefault('registered_builtin_methods', {})
-        dct[method_name] = func
-        return func
-    return wrapper
-
-
-@has_builtin_methods
-class GeneratorMixin(object):
-    array_type = None
-
-    @register_builtin_method('send')
-    @register_builtin_method('next', python_version_match=2)
-    @register_builtin_method('__next__', python_version_match=3)
-    def py__next__(self):
-        # TODO add TypeError if params are given.
-        return unite(lazy_context.infer() for lazy_context in self.py__iter__())
-
-    def get_filters(self, search_global, until_position=None, origin_scope=None):
-        gen_obj = compiled.get_special_object(self.evaluator, 'GENERATOR_OBJECT')
-        yield SpecialMethodFilter(self, self.builtin_methods, gen_obj)
-        for filter in gen_obj.get_filters(search_global):
-            yield filter
-
-    def py__bool__(self):
-        return True
-
-    def py__class__(self):
-        gen_obj = compiled.get_special_object(self.evaluator, 'GENERATOR_OBJECT')
-        return gen_obj.py__class__()
-
-    @property
-    def name(self):
-        return compiled.CompiledContextName(self, 'generator')
-
-
-class Generator(GeneratorMixin, context.Context):
-    """Handling of `yield` functions."""
-
-    def __init__(self, evaluator, func_execution_context):
-        super(Generator, self).__init__(evaluator, parent_context=evaluator.BUILTINS)
-        self._func_execution_context = func_execution_context
-
-    def py__iter__(self):
-        return self._func_execution_context.get_yield_values()
-
-    def __repr__(self):
-        return "<%s of %s>" % (type(self).__name__, self._func_execution_context)
-
-
-class CompForContext(context.TreeContext):
-    @classmethod
-    def from_comp_for(cls, parent_context, comp_for):
-        return cls(parent_context.evaluator, parent_context, comp_for)
-
-    def __init__(self, evaluator, parent_context, comp_for):
-        super(CompForContext, self).__init__(evaluator, parent_context)
-        self.tree_node = comp_for
-
-    def get_node(self):
-        return self.tree_node
-
-    def get_filters(self, search_global, until_position=None, origin_scope=None):
-        yield ParserTreeFilter(self.evaluator, self)
-
-
-class Comprehension(AbstractSequence):
-    @staticmethod
-    def from_atom(evaluator, context, atom):
-        bracket = atom.children[0]
-        if bracket == '{':
-            if atom.children[1].children[1] == ':':
-                cls = DictComprehension
-            else:
-                cls = SetComprehension
-        elif bracket == '(':
-            cls = GeneratorComprehension
-        elif bracket == '[':
-            cls = ListComprehension
-        return cls(evaluator, context, atom)
-
-    def __init__(self, evaluator, defining_context, atom):
-        super(Comprehension, self).__init__(evaluator)
-        self._defining_context = defining_context
-        self._atom = atom
-
-    def _get_comprehension(self):
-        # The atom contains a testlist_comp
-        return self._atom.children[1]
-
-    def _get_comp_for(self):
-        # The atom contains a testlist_comp
-        return self._get_comprehension().children[1]
-
-    def _eval_node(self, index=0):
-        """
-        The first part `x + 1` of the list comprehension:
-
-            [x + 1 for x in foo]
-        """
-        return self._get_comprehension().children[index]
-
-    @memoize_default()
-    def _get_comp_for_context(self, parent_context, comp_for):
-        # TODO shouldn't this be part of create_context?
-        return CompForContext.from_comp_for(parent_context, comp_for)
-
-    def _nested(self, comp_fors, parent_context=None):
-        evaluator = self.evaluator
-        comp_for = comp_fors[0]
-        input_node = comp_for.children[3]
-        parent_context = parent_context or self._defining_context
-        input_types = parent_context.eval_node(input_node)
-
-        iterated = py__iter__(evaluator, input_types, input_node)
-        exprlist = comp_for.children[1]
-        for i, lazy_context in enumerate(iterated):
-            types = lazy_context.infer()
-            dct = unpack_tuple_to_dict(evaluator, types, exprlist)
-            context = self._get_comp_for_context(
-                parent_context,
-                comp_for,
-            )
-            with helpers.predefine_names(context, comp_for, dct):
-                try:
-                    for result in self._nested(comp_fors[1:], context):
-                        yield result
-                except IndexError:
-                    iterated = context.eval_node(self._eval_node())
-                    if self.array_type == 'dict':
-                        yield iterated, context.eval_node(self._eval_node(2))
-                    else:
-                        yield iterated
-
-    @memoize_default(default=[])
-    @common.to_list
-    def _iterate(self):
-        comp_fors = tuple(self._get_comp_for().get_comp_fors())
-        for result in self._nested(comp_fors):
-            yield result
-
-    def py__iter__(self):
-        for set_ in self._iterate():
-            yield context.LazyKnownContexts(set_)
-
-    def __repr__(self):
-        return "<%s of %s>" % (type(self).__name__, self._atom)
-
-
-class ArrayMixin(object):
-    def get_filters(self, search_global, until_position=None, origin_scope=None):
-        # `array.type` is a string with the type, e.g. 'list'.
-        compiled_obj = compiled.builtin_from_name(self.evaluator, self.array_type)
-        yield SpecialMethodFilter(self, self.builtin_methods, compiled_obj)
-        for typ in compiled_obj.execute_evaluated(self):
-            for filter in typ.get_filters():
-                yield filter
-
-    def py__bool__(self):
-        return None  # We don't know the length, because of appends.
-
-    def py__class__(self):
-        return compiled.builtin_from_name(self.evaluator, self.array_type)
-
-    @safe_property
-    def parent(self):
-        return self.evaluator.BUILTINS
-
-    def dict_values(self):
-        return unite(self._defining_context.eval_node(v) for k, v in self._items())
-
-
-class ListComprehension(ArrayMixin, Comprehension):
-    array_type = 'list'
-
-    def py__getitem__(self, index):
-        if isinstance(index, slice):
-            return set([self])
-
-        all_types = list(self.py__iter__())
-        return all_types[index].infer()
-
-
-class SetComprehension(ArrayMixin, Comprehension):
-    array_type = 'set'
-
-
-@has_builtin_methods
-class DictComprehension(ArrayMixin, Comprehension):
-    array_type = 'dict'
-
-    def _get_comp_for(self):
-        return self._get_comprehension().children[3]
-
-    def py__iter__(self):
-        for keys, values in self._iterate():
-            yield context.LazyKnownContexts(keys)
-
-    def py__getitem__(self, index):
-        for keys, values in self._iterate():
-            for k in keys:
-                if isinstance(k, compiled.CompiledObject):
-                    if k.obj == index:
-                        return values
-        return self.dict_values()
-
-    def dict_values(self):
-        return unite(values for keys, values in self._iterate())
-
-    @register_builtin_method('values')
-    def _imitate_values(self):
-        lazy_context = context.LazyKnownContexts(self.dict_values())
-        return set([FakeSequence(self.evaluator, 'list', [lazy_context])])
-
-    @register_builtin_method('items')
-    def _imitate_items(self):
-        items = set(
-            FakeSequence(
-                self.evaluator, 'tuple'
-                (context.LazyKnownContexts(keys), context.LazyKnownContexts(values))
-            ) for keys, values in self._iterate()
-        )
-
-        return create_evaluated_sequence_set(self.evaluator, items, sequence_type='list')
-
-
-class GeneratorComprehension(GeneratorMixin, Comprehension):
-    pass
-
-
-class SequenceLiteralContext(ArrayMixin, AbstractSequence):
-    mapping = {'(': 'tuple',
-               '[': 'list',
-               '{': 'set'}
-
-    def __init__(self, evaluator, defining_context, atom):
-        super(SequenceLiteralContext, self).__init__(evaluator)
-        self.atom = atom
-        self._defining_context = defining_context
-
-        if self.atom.type in ('testlist_star_expr', 'testlist'):
-            self.array_type = 'tuple'
-        else:
-            self.array_type = SequenceLiteralContext.mapping[atom.children[0]]
-            """The builtin name of the array (list, set, tuple or dict)."""
-
-    def py__getitem__(self, index):
-        """Here the index is an int/str. Raises IndexError/KeyError."""
-        if self.array_type == 'dict':
-            for key, value in self._items():
-                for k in self._defining_context.eval_node(key):
-                    if isinstance(k, compiled.CompiledObject) \
-                            and index == k.obj:
-                        return self._defining_context.eval_node(value)
-            raise KeyError('No key found in dictionary %s.' % self)
-
-        # Can raise an IndexError
-        if isinstance(index, slice):
-            return set([self])
-        else:
-            return self._defining_context.eval_node(self._items()[index])
-
-    def py__iter__(self):
-        """
-        While values returns the possible values for any array field, this
-        function returns the value for a certain index.
-        """
-        if self.array_type == 'dict':
-            # Get keys.
-            types = set()
-            for k, _ in self._items():
-                types |= self._defining_context.eval_node(k)
-            # We don't know which dict index comes first, therefore always
-            # yield all the types.
-            for _ in types:
-                yield context.LazyKnownContexts(types)
-        else:
-            for node in self._items():
-                yield context.LazyTreeContext(self._defining_context, node)
-
-            for addition in check_array_additions(self._defining_context, self):
-                yield addition
-
-    def _values(self):
-        """Returns a list of a list of node."""
-        if self.array_type == 'dict':
-            return unite(v for k, v in self._items())
-        else:
-            return self._items()
-
-    def _items(self):
-        c = self.atom.children
-
-        if self.atom.type in ('testlist_star_expr', 'testlist'):
-            return c[::2]
-
-        array_node = c[1]
-        if array_node in (']', '}', ')'):
-            return []  # Direct closing bracket, doesn't contain items.
-
-        if array_node.type == 'testlist_comp':
-            return array_node.children[::2]
-        elif array_node.type == 'dictorsetmaker':
-            kv = []
-            iterator = iter(array_node.children)
-            for key in iterator:
-                op = next(iterator, None)
-                if op is None or op == ',':
-                    kv.append(key)  # A set.
-                else:
-                    assert op == ':'  # A dict.
-                    kv.append((key, next(iterator)))
-                    next(iterator, None)  # Possible comma.
-            return kv
-        else:
-            return [array_node]
-
-    def exact_key_items(self):
-        """
-        Returns a generator of tuples like dict.items(), where the key is
-        resolved (as a string) and the values are still lazy contexts.
-        """
-        for key_node, value in self._items():
-            for key in self._defining_context.eval_node(key_node):
-                if precedence.is_string(key):
-                    yield key.obj, context.LazyTreeContext(self._defining_context, value)
-
-    def __repr__(self):
-        return "<%s of %s>" % (self.__class__.__name__, self.atom)
-
-
-@has_builtin_methods
-class DictLiteralContext(SequenceLiteralContext):
-    array_type = 'dict'
-
-    def __init__(self, evaluator, defining_context, atom):
-        super(SequenceLiteralContext, self).__init__(evaluator)
-        self._defining_context = defining_context
-        self.atom = atom
-
-    @register_builtin_method('values')
-    def _imitate_values(self):
-        lazy_context = context.LazyKnownContexts(self.dict_values())
-        return set([FakeSequence(self.evaluator, 'list', [lazy_context])])
-
-    @register_builtin_method('items')
-    def _imitate_items(self):
-        lazy_contexts = [
-            context.LazyKnownContext(FakeSequence(
-                self.evaluator, 'tuple',
-                (context.LazyTreeContext(self._defining_context, key_node),
-                 context.LazyTreeContext(self._defining_context, value_node))
-            )) for key_node, value_node in self._items()
-        ]
-
-        return set([FakeSequence(self.evaluator, 'list', lazy_contexts)])
-
-
-class _FakeArray(SequenceLiteralContext):
-    def __init__(self, evaluator, container, type):
-        super(SequenceLiteralContext, self).__init__(evaluator)
-        self.array_type = type
-        self.atom = container
-        # TODO is this class really needed?
-
-
-class ImplicitTuple(_FakeArray):
-    def __init__(self, evaluator, testlist):
-        super(ImplicitTuple, self).__init__(evaluator, testlist, 'tuple')
-        raise NotImplementedError
-        self._testlist = testlist
-
-    def _items(self):
-        return self._testlist.children[::2]
-
-
-class FakeSequence(_FakeArray):
-    def __init__(self, evaluator, array_type, lazy_context_list):
-        """
-        type should be one of "tuple", "list"
-        """
-        super(FakeSequence, self).__init__(evaluator, None, array_type)
-        self._lazy_context_list = lazy_context_list
-
-    def _items(self):
-        raise DeprecationWarning
-        return self._context_list
-
-    def py__getitem__(self, index):
-        return set(self._lazy_context_list[index].infer())
-
-    def py__iter__(self):
-        return self._lazy_context_list
-
-    def __repr__(self):
-        return "<%s of %s>" % (type(self).__name__, self._lazy_context_list)
-
-
-class FakeDict(_FakeArray):
-    def __init__(self, evaluator, dct):
-        super(FakeDict, self).__init__(evaluator, dct, 'dict')
-        self._dct = dct
-
-    def py__iter__(self):
-        for key in self._dct:
-            yield context.LazyKnownContext(compiled.create(self.evaluator, key))
-
-    def py__getitem__(self, index):
-        return self._dct[index].infer()
-
-    def dict_values(self):
-        return unite(lazy_context.infer() for lazy_context in self._dct.values())
-
-    def _items(self):
-        raise DeprecationWarning
-        for key, values in self._dct.items():
-            # TODO this is not proper. The values could be multiple values?!
-            yield key, values[0]
-
-    def exact_key_items(self):
-        return self._dct.items()
-
-
-class MergedArray(_FakeArray):
-    def __init__(self, evaluator, arrays):
-        super(MergedArray, self).__init__(evaluator, arrays, arrays[-1].array_type)
-        self._arrays = arrays
-
-    def py__iter__(self):
-        for array in self._arrays:
-            for lazy_context in array.py__iter__():
-                yield lazy_context
-
-    def py__getitem__(self, index):
-        return unite(lazy_context.infer() for lazy_context in self.py__iter__())
-
-    def _items(self):
-        for array in self._arrays:
-            for a in array._items():
-                yield a
-
-    def __len__(self):
-        return sum(len(a) for a in self._arrays)
-
-
-def unpack_tuple_to_dict(evaluator, types, exprlist):
-    """
-    Unpacking tuple assignments in for statements and expr_stmts.
-    """
-    if exprlist.type == 'name':
-        return {exprlist.value: types}
-    elif exprlist.type == 'atom' and exprlist.children[0] in '([':
-        return unpack_tuple_to_dict(evaluator, types, exprlist.children[1])
-    elif exprlist.type in ('testlist', 'testlist_comp', 'exprlist',
-                           'testlist_star_expr'):
-        dct = {}
-        parts = iter(exprlist.children[::2])
-        n = 0
-        for lazy_context in py__iter__(evaluator, types, exprlist):
-            n += 1
-            try:
-                part = next(parts)
-            except StopIteration:
-                # TODO this context is probably not right.
-                analysis.add(next(iter(types)), 'value-error-too-many-values', part,
-                             message="ValueError: too many values to unpack (expected %s)" % n)
-            else:
-                dct.update(unpack_tuple_to_dict(evaluator, lazy_context.infer(), part))
-        has_parts = next(parts, None)
-        if types and has_parts is not None:
-            # TODO this context is probably not right.
-            analysis.add(next(iter(types)), 'value-error-too-few-values', has_parts,
-                         message="ValueError: need more than %s values to unpack" % n)
-        return dct
-    elif exprlist.type == 'power' or exprlist.type == 'atom_expr':
-        # Something like ``arr[x], var = ...``.
-        # This is something that is not yet supported, would also be difficult
-        # to write into a dict.
-        return {}
-    elif exprlist.type == 'star_expr':  # `a, *b, c = x` type unpackings
-        # Currently we're not supporting them.
-        return {}
-    raise NotImplementedError
-
-
-def py__iter__(evaluator, types, node=None):
-    debug.dbg('py__iter__')
-    type_iters = []
-    for typ in types:
-        try:
-            iter_method = typ.py__iter__
-        except AttributeError:
-            if node is not None:
-                # TODO this context is probably not right.
-                analysis.add(typ, 'type-error-not-iterable', node,
-                             message="TypeError: '%s' object is not iterable" % typ)
-        else:
-            type_iters.append(iter_method())
-
-    for lazy_contexts in zip_longest(*type_iters):
-        yield context.get_merged_lazy_context(
-            [l for l in lazy_contexts if l is not None]
-        )
-
-
-def py__iter__types(evaluator, types, node=None):
-    """
-    Calls `py__iter__`, but ignores the ordering in the end and just returns
-    all types that it contains.
-    """
-    return unite(lazy_context.infer() for lazy_context in py__iter__(evaluator, types, node))
-
-
-def py__getitem__(evaluator, context, types, trailer):
-    from jedi.evaluate.representation import ClassContext
-    from jedi.evaluate.instance import TreeInstance
-    result = set()
-
-    trailer_op, node, trailer_cl = trailer.children
-    assert trailer_op == "["
-    assert trailer_cl == "]"
-
-    # special case: PEP0484 typing module, see
-    # https://github.com/davidhalter/jedi/issues/663
-    for typ in list(types):
-        if isinstance(typ, (ClassContext, TreeInstance)):
-            typing_module_types = pep0484.py__getitem__(context, typ, node)
-            if typing_module_types is not None:
-                types.remove(typ)
-                result |= typing_module_types
-
-    if not types:
-        # all consumed by special cases
-        return result
-
-    for index in create_index_types(evaluator, context, node):
-        if isinstance(index, (compiled.CompiledObject, Slice)):
-            index = index.obj
-
-        if type(index) not in (float, int, str, unicode, slice):
-            # If the index is not clearly defined, we have to get all the
-            # possiblities.
-            for typ in list(types):
-                if isinstance(typ, AbstractSequence) and typ.array_type == 'dict':
-                    types.remove(typ)
-                    result |= typ.dict_values()
-            return result | py__iter__types(evaluator, types)
-
-        for typ in types:
-            # The actual getitem call.
-            try:
-                getitem = typ.py__getitem__
-            except AttributeError:
-                # TODO this context is probably not right.
-                analysis.add(context, 'type-error-not-subscriptable', trailer_op,
-                             message="TypeError: '%s' object is not subscriptable" % typ)
-            else:
-                try:
-                    result |= getitem(index)
-                except IndexError:
-                    result |= py__iter__types(evaluator, set([typ]))
-                except KeyError:
-                    # Must be a dict. Lists don't raise KeyErrors.
-                    result |= typ.dict_values()
-    return result
-
-
-def check_array_additions(context, sequence):
-    """ Just a mapper function for the internal _check_array_additions """
-    if sequence.array_type not in ('list', 'set'):
-        # TODO also check for dict updates
-        return set()
-
-    return _check_array_additions(context, sequence)
-
-
-@memoize_default(default=set())
-@debug.increase_indent
-def _check_array_additions(context, sequence):
-    """
-    Checks if a `Array` has "add" (append, insert, extend) statements:
-
-    >>> a = [""]
-    >>> a.append(1)
-    """
-    from jedi.evaluate import param
-
-    debug.dbg('Dynamic array search for %s' % sequence, color='MAGENTA')
-    module_context = context.get_root_context()
-    if not settings.dynamic_array_additions or isinstance(module_context, compiled.CompiledObject):
-        debug.dbg('Dynamic array search aborted.', color='MAGENTA')
-        return set()
-
-    def find_additions(context, arglist, add_name):
-        params = list(param.TreeArguments(context.evaluator, context, arglist).unpack())
-        result = set()
-        if add_name in ['insert']:
-            params = params[1:]
-        if add_name in ['append', 'add', 'insert']:
-            for key, lazy_context in params:
-                result.add(lazy_context)
-        elif add_name in ['extend', 'update']:
-            for key, lazy_context in params:
-                result |= set(py__iter__(context.evaluator, lazy_context.infer()))
-        return result
-
-    temp_param_add, settings.dynamic_params_for_other_modules = \
-        settings.dynamic_params_for_other_modules, False
-
-    is_list = sequence.name.string_name == 'list'
-    search_names = (['append', 'extend', 'insert'] if is_list else ['add', 'update'])
-
-    added_types = set()
-    for add_name in search_names:
-        try:
-            possible_names = module_context.tree_node.used_names[add_name]
-        except KeyError:
-            continue
-        else:
-            for name in possible_names:
-                context_node = context.tree_node
-                if not (context_node.start_pos < name.start_pos < context_node.end_pos):
-                    continue
-                trailer = name.parent
-                power = trailer.parent
-                trailer_pos = power.children.index(trailer)
-                try:
-                    execution_trailer = power.children[trailer_pos + 1]
-                except IndexError:
-                    continue
-                else:
-                    if execution_trailer.type != 'trailer' \
-                            or execution_trailer.children[0] != '(' \
-                            or execution_trailer.children[1] == ')':
-                        continue
-
-                random_context = context.create_context(name)
-
-                with recursion.execution_allowed(context.evaluator, power) as allowed:
-                    if allowed:
-                        found = helpers.evaluate_call_of_leaf(
-                            random_context,
-                            name,
-                            cut_own_trailer=True
-                        )
-                        if sequence in found:
-                            # The arrays match. Now add the results
-                            added_types |= find_additions(
-                                random_context,
-                                execution_trailer.children[1],
-                                add_name
-                            )
-
-    # reset settings
-    settings.dynamic_params_for_other_modules = temp_param_add
-    debug.dbg('Dynamic array result %s' % added_types, color='MAGENTA')
-    return added_types
-
-
-def get_dynamic_array_instance(instance):
-    """Used for set() and list() instances."""
-    if not settings.dynamic_array_additions:
-        return instance.var_args
-
-    ai = _ArrayInstance(instance)
-    from jedi.evaluate import param
-    return param.ValuesArguments([[ai]])
-
-
-class _ArrayInstance(object):
-    """
-    Used for the usage of set() and list().
-    This is definitely a hack, but a good one :-)
-    It makes it possible to use set/list conversions.
-
-    In contrast to Array, ListComprehension and all other iterable types, this
-    is something that is only used inside `evaluate/compiled/fake/builtins.py`
-    and therefore doesn't need filters, `py__bool__` and so on, because
-    we don't use these operations in `builtins.py`.
-    """
-    def __init__(self, instance):
-        self.instance = instance
-        self.var_args = instance.var_args
-
-    def py__iter__(self):
-        var_args = self.var_args
-        try:
-            _, lazy_context = next(var_args.unpack())
-        except StopIteration:
-            pass
-        else:
-            for lazy in py__iter__(self.instance.evaluator, lazy_context.infer()):
-                yield lazy
-
-        from jedi.evaluate import param
-        if isinstance(var_args, param.TreeArguments):
-            additions = _check_array_additions(var_args.context, self.instance)
-            for addition in additions:
-                yield addition
-
-
-class Slice(context.Context):
-    def __init__(self, context, start, stop, step):
-        super(Slice, self).__init__(
-            context.evaluator,
-            parent_context=context.evaluator.BUILTINS
-        )
-        self._context = context
-        # all of them are either a Precedence or None.
-        self._start = start
-        self._stop = stop
-        self._step = step
-
-    @property
-    def obj(self):
-        """
-        Imitate CompiledObject.obj behavior and return a ``builtin.slice()``
-        object.
-        """
-        def get(element):
-            if element is None:
-                return None
-
-            result = self._context.eval_node(element)
-            if len(result) != 1:
-                # For simplicity, we want slices to be clear defined with just
-                # one type.  Otherwise we will return an empty slice object.
-                raise IndexError
-            try:
-                return list(result)[0].obj
-            except AttributeError:
-                return None
-
-        try:
-            return slice(get(self._start), get(self._stop), get(self._step))
-        except IndexError:
-            return slice(None, None, None)
-
-
-def create_index_types(evaluator, context, index):
-    """
-    Handles slices in subscript nodes.
-    """
-    if index == ':':
-        # Like array[:]
-        return set([Slice(context, None, None, None)])
-    elif index.type == 'subscript':  # subscript is a slice operation.
-        # Like array[:3]
-        result = []
-        for el in index.children:
-            if el == ':':
-                if not result:
-                    result.append(None)
-            elif el.type == 'sliceop':
-                if len(el.children) == 2:
-                    result.append(el.children[1])
-            else:
-                result.append(el)
-        result += [None] * (3 - len(result))
-
-        return set([Slice(context, *result)])
-
-    # No slices
-    return context.eval_node(index)

jedi/evaluate/jedi_typing.py

@@ -1,100 +0,0 @@
-"""
-This module is not intended to be used in jedi, rather it will be fed to the
-jedi-parser to replace classes in the typing module
-"""
-
-try:
-    from collections import abc
-except ImportError:
-    # python 2
-    import collections as abc
-
-
-def factory(typing_name, indextypes):
-    class Iterable(abc.Iterable):
-        def __iter__(self):
-            while True:
-                yield indextypes[0]()
-
-    class Iterator(Iterable, abc.Iterator):
-        def next(self):
-            """ needed for python 2 """
-            return self.__next__()
-
-        def __next__(self):
-            return indextypes[0]()
-
-    class Sequence(abc.Sequence):
-        def __getitem__(self, index):
-            return indextypes[0]()
-
-    class MutableSequence(Sequence, abc.MutableSequence):
-        pass
-
-    class List(MutableSequence, list):
-        pass
-
-    class Tuple(Sequence, tuple):
-        def __getitem__(self, index):
-            if indextypes[1] == Ellipsis:
-                # https://www.python.org/dev/peps/pep-0484/#the-typing-module
-                # Tuple[int, ...] means a tuple of ints of indetermined length
-                return indextypes[0]()
-            else:
-                return indextypes[index]()
-
-    class AbstractSet(Iterable, abc.Set):
-        pass
-
-    class MutableSet(AbstractSet, abc.MutableSet):
-        pass
-
-    class KeysView(Iterable, abc.KeysView):
-        pass
-
-    class ValuesView(abc.ValuesView):
-        def __iter__(self):
-            while True:
-                yield indextypes[1]()
-
-    class ItemsView(abc.ItemsView):
-        def __iter__(self):
-            while True:
-                yield (indextypes[0](), indextypes[1]())
-
-    class Mapping(Iterable, abc.Mapping):
-        def __getitem__(self, item):
-            return indextypes[1]()
-
-        def keys(self):
-            return KeysView()
-
-        def values(self):
-            return ValuesView()
-
-        def items(self):
-            return ItemsView()
-
-    class MutableMapping(Mapping, abc.MutableMapping):
-        pass
-
-    class Dict(MutableMapping, dict):
-        pass
-
-    dct = {
-        "Sequence": Sequence,
-        "MutableSequence": MutableSequence,
-        "List": List,
-        "Iterable": Iterable,
-        "Iterator": Iterator,
-        "AbstractSet": AbstractSet,
-        "MutableSet": MutableSet,
-        "Mapping": Mapping,
-        "MutableMapping": MutableMapping,
-        "Tuple": Tuple,
-        "KeysView": KeysView,
-        "ItemsView": ItemsView,
-        "ValuesView": ValuesView,
-        "Dict": Dict,
-    }
-    return dct[typing_name]

jedi/evaluate/lazy_context.py

@@ -0,0 +1,59 @@
+from jedi.evaluate.base_context import ContextSet, NO_CONTEXTS
+from jedi.common.utils import monkeypatch
+
+
+class AbstractLazyContext(object):
+    def __init__(self, data):
+        self.data = data
+
+    def __repr__(self):
+        return '<%s: %s>' % (self.__class__.__name__, self.data)
+
+    def infer(self):
+        raise NotImplementedError
+
+
+class LazyKnownContext(AbstractLazyContext):
+    """data is a context."""
+    def infer(self):
+        return ContextSet([self.data])
+
+
+class LazyKnownContexts(AbstractLazyContext):
+    """data is a ContextSet."""
+    def infer(self):
+        return self.data
+
+
+class LazyUnknownContext(AbstractLazyContext):
+    def __init__(self):
+        super(LazyUnknownContext, self).__init__(None)
+
+    def infer(self):
+        return NO_CONTEXTS
+
+
+class LazyTreeContext(AbstractLazyContext):
+    def __init__(self, context, node):
+        super(LazyTreeContext, self).__init__(node)
+        self.context = context
+        # We need to save the predefined names. It's an unfortunate side effect
+        # that needs to be tracked otherwise results will be wrong.
+        self._predefined_names = dict(context.predefined_names)
+
+    def infer(self):
+        with monkeypatch(self.context, 'predefined_names', self._predefined_names):
+            return self.context.eval_node(self.data)
+
+
+def get_merged_lazy_context(lazy_contexts):
+    if len(lazy_contexts) > 1:
+        return MergedLazyContexts(lazy_contexts)
+    else:
+        return lazy_contexts[0]
+
+
+class MergedLazyContexts(AbstractLazyContext):
+    """data is a list of lazy contexts."""
+    def infer(self):
+        return ContextSet.from_sets(l.infer() for l in self.data)

jedi/evaluate/names.py

@@ -0,0 +1,375 @@
+from abc import abstractmethod
+
+from parso.tree import search_ancestor
+
+from jedi._compatibility import Parameter
+from jedi.evaluate.base_context import ContextSet, NO_CONTEXTS
+from jedi.cache import memoize_method
+
+
+class AbstractNameDefinition(object):
+    start_pos = None
+    string_name = None
+    parent_context = None
+    tree_name = None
+    is_context_name = True
+    """
+    Used for the Jedi API to know if it's a keyword or an actual name.
+    """
+
+    @abstractmethod
+    def infer(self):
+        raise NotImplementedError
+
+    @abstractmethod
+    def goto(self):
+        # Typically names are already definitions and therefore a goto on that
+        # name will always result on itself.
+        return {self}
+
+    def get_qualified_names(self, include_module_names=False):
+        qualified_names = self._get_qualified_names()
+        if qualified_names is None or not include_module_names:
+            return qualified_names
+
+        module_names = self.get_root_context().string_names
+        if module_names is None:
+            return None
+        return module_names + qualified_names
+
+    def _get_qualified_names(self):
+        # By default, a name has no qualified names.
+        return None
+
+    def get_root_context(self):
+        return self.parent_context.get_root_context()
+
+    def __repr__(self):
+        if self.start_pos is None:
+            return '<%s: string_name=%s>' % (self.__class__.__name__, self.string_name)
+        return '<%s: string_name=%s start_pos=%s>' % (self.__class__.__name__,
+                                                      self.string_name, self.start_pos)
+
+    def is_import(self):
+        return False
+
+    @property
+    def api_type(self):
+        return self.parent_context.api_type
+
+
+class AbstractArbitraryName(AbstractNameDefinition):
+    """
+    When you e.g. want to complete dicts keys, you probably want to complete
+    string literals, which is not really a name, but for Jedi we use this
+    concept of Name for completions as well.
+    """
+    is_context_name = False
+
+    def __init__(self, evaluator, string):
+        self.evaluator = evaluator
+        self.string_name = string
+        self.parent_context = evaluator.builtins_module
+
+    def infer(self):
+        return NO_CONTEXTS
+
+
+class AbstractTreeName(AbstractNameDefinition):
+    def __init__(self, parent_context, tree_name):
+        self.parent_context = parent_context
+        self.tree_name = tree_name
+
+    def get_qualified_names(self, include_module_names=False):
+        import_node = search_ancestor(self.tree_name, 'import_name', 'import_from')
+        # For import nodes we cannot just have names, because it's very unclear
+        # how they would look like. For now we just ignore them in most cases.
+        # In case of level == 1, it works always, because it's like a submodule
+        # lookup.
+        if import_node is not None and not (import_node.level == 1
+                                            and self.get_root_context().is_package):
+            # TODO improve the situation for when level is present.
+            if include_module_names and not import_node.level:
+                return tuple(n.value for n in import_node.get_path_for_name(self.tree_name))
+            else:
+                return None
+
+        return super(AbstractTreeName, self).get_qualified_names(include_module_names)
+
+    def _get_qualified_names(self):
+        parent_names = self.parent_context.get_qualified_names()
+        if parent_names is None:
+            return None
+        return parent_names + (self.tree_name.value,)
+
+    def goto(self, **kwargs):
+        return self.parent_context.evaluator.goto(self.parent_context, self.tree_name, **kwargs)
+
+    def is_import(self):
+        imp = search_ancestor(self.tree_name, 'import_from', 'import_name')
+        return imp is not None
+
+    @property
+    def string_name(self):
+        return self.tree_name.value
+
+    @property
+    def start_pos(self):
+        return self.tree_name.start_pos
+
+
+class ContextNameMixin(object):
+    def infer(self):
+        return ContextSet([self._context])
+
+    def _get_qualified_names(self):
+        return self._context.get_qualified_names()
+
+    def get_root_context(self):
+        if self.parent_context is None:  # A module
+            return self._context
+        return super(ContextNameMixin, self).get_root_context()
+
+    @property
+    def api_type(self):
+        return self._context.api_type
+
+
+class ContextName(ContextNameMixin, AbstractTreeName):
+    def __init__(self, context, tree_name):
+        super(ContextName, self).__init__(context.parent_context, tree_name)
+        self._context = context
+
+    def goto(self):
+        return ContextSet([self._context.name])
+
+
+class TreeNameDefinition(AbstractTreeName):
+    _API_TYPES = dict(
+        import_name='module',
+        import_from='module',
+        funcdef='function',
+        param='param',
+        classdef='class',
+    )
+
+    def infer(self):
+        # Refactor this, should probably be here.
+        from jedi.evaluate.syntax_tree import tree_name_to_contexts
+        parent = self.parent_context
+        return tree_name_to_contexts(parent.evaluator, parent, self.tree_name)
+
+    @property
+    def api_type(self):
+        definition = self.tree_name.get_definition(import_name_always=True)
+        if definition is None:
+            return 'statement'
+        return self._API_TYPES.get(definition.type, 'statement')
+
+
+class _ParamMixin(object):
+    def maybe_positional_argument(self, include_star=True):
+        options = [Parameter.POSITIONAL_ONLY, Parameter.POSITIONAL_OR_KEYWORD]
+        if include_star:
+            options.append(Parameter.VAR_POSITIONAL)
+        return self.get_kind() in options
+
+    def maybe_keyword_argument(self, include_stars=True):
+        options = [Parameter.KEYWORD_ONLY, Parameter.POSITIONAL_OR_KEYWORD]
+        if include_stars:
+            options.append(Parameter.VAR_KEYWORD)
+        return self.get_kind() in options
+
+    def _kind_string(self):
+        kind = self.get_kind()
+        if kind == Parameter.VAR_POSITIONAL:  # *args
+            return '*'
+        if kind == Parameter.VAR_KEYWORD:  # **kwargs
+            return '**'
+        return ''
+
+
+class ParamNameInterface(_ParamMixin):
+    api_type = u'param'
+
+    def get_kind(self):
+        raise NotImplementedError
+
+    def to_string(self):
+        raise NotImplementedError
+
+    def get_param(self):
+        # TODO document better where this is used and when. Currently it has
+        #      very limited use, but is still in use. It's currently not even
+        #      clear what values would be allowed.
+        return None
+
+    @property
+    def star_count(self):
+        kind = self.get_kind()
+        if kind == Parameter.VAR_POSITIONAL:
+            return 1
+        if kind == Parameter.VAR_KEYWORD:
+            return 2
+        return 0
+
+
+class BaseTreeParamName(ParamNameInterface, AbstractTreeName):
+    annotation_node = None
+    default_node = None
+
+    def to_string(self):
+        output = self._kind_string() + self.string_name
+        annotation = self.annotation_node
+        default = self.default_node
+        if annotation is not None:
+            output += ': ' + annotation.get_code(include_prefix=False)
+        if default is not None:
+            output += '=' + default.get_code(include_prefix=False)
+        return output
+
+
+class ParamName(BaseTreeParamName):
+    def _get_param_node(self):
+        return search_ancestor(self.tree_name, 'param')
+
+    @property
+    def annotation_node(self):
+        return self._get_param_node().annotation
+
+    def infer_annotation(self, execute_annotation=True):
+        node = self.annotation_node
+        if node is None:
+            return NO_CONTEXTS
+        contexts = self.parent_context.parent_context.eval_node(node)
+        if execute_annotation:
+            contexts = contexts.execute_annotation()
+        return contexts
+
+    def infer_default(self):
+        node = self.default_node
+        if node is None:
+            return NO_CONTEXTS
+        return self.parent_context.parent_context.eval_node(node)
+
+    @property
+    def default_node(self):
+        return self._get_param_node().default
+
+    @property
+    def string_name(self):
+        name = self.tree_name.value
+        if name.startswith('__'):
+            # Params starting with __ are an equivalent to positional only
+            # variables in typeshed.
+            name = name[2:]
+        return name
+
+    def get_kind(self):
+        tree_param = self._get_param_node()
+        if tree_param.star_count == 1:  # *args
+            return Parameter.VAR_POSITIONAL
+        if tree_param.star_count == 2:  # **kwargs
+            return Parameter.VAR_KEYWORD
+
+        # Params starting with __ are an equivalent to positional only
+        # variables in typeshed.
+        if tree_param.name.value.startswith('__'):
+            return Parameter.POSITIONAL_ONLY
+
+        parent = tree_param.parent
+        param_appeared = False
+        for p in parent.children:
+            if param_appeared:
+                if p == '/':
+                    return Parameter.POSITIONAL_ONLY
+            else:
+                if p == '*':
+                    return Parameter.KEYWORD_ONLY
+                if p.type == 'param':
+                    if p.star_count:
+                        return Parameter.KEYWORD_ONLY
+                    if p == tree_param:
+                        param_appeared = True
+        return Parameter.POSITIONAL_OR_KEYWORD
+
+    def infer(self):
+        return self.get_param().infer()
+
+    def get_param(self):
+        params, _ = self.parent_context.get_executed_params_and_issues()
+        param_node = search_ancestor(self.tree_name, 'param')
+        return params[param_node.position_index]
+
+
+class ParamNameWrapper(_ParamMixin):
+    def __init__(self, param_name):
+        self._wrapped_param_name = param_name
+
+    def __getattr__(self, name):
+        return getattr(self._wrapped_param_name, name)
+
+    def __repr__(self):
+        return '<%s: %s>' % (self.__class__.__name__, self._wrapped_param_name)
+
+
+class ImportName(AbstractNameDefinition):
+    start_pos = (1, 0)
+    _level = 0
+
+    def __init__(self, parent_context, string_name):
+        self._from_module_context = parent_context
+        self.string_name = string_name
+
+    def get_qualified_names(self, include_module_names=False):
+        if include_module_names:
+            if self._level:
+                assert self._level == 1, "Everything else is not supported for now"
+                module_names = self._from_module_context.string_names
+                if module_names is None:
+                    return module_names
+                return module_names + (self.string_name,)
+            return (self.string_name,)
+        return ()
+
+    @property
+    def parent_context(self):
+        m = self._from_module_context
+        import_contexts = self.infer()
+        if not import_contexts:
+            return m
+        # It's almost always possible to find the import or to not find it. The
+        # importing returns only one context, pretty much always.
+        return next(iter(import_contexts))
+
+    @memoize_method
+    def infer(self):
+        from jedi.evaluate.imports import Importer
+        m = self._from_module_context
+        return Importer(m.evaluator, [self.string_name], m, level=self._level).follow()
+
+    def goto(self):
+        return [m.name for m in self.infer()]
+
+    @property
+    def api_type(self):
+        return 'module'
+
+
+class SubModuleName(ImportName):
+    _level = 1
+
+
+class NameWrapper(object):
+    def __init__(self, wrapped_name):
+        self._wrapped_name = wrapped_name
+
+    @abstractmethod
+    def infer(self):
+        raise NotImplementedError
+
+    def __getattr__(self, name):
+        return getattr(self._wrapped_name, name)
+
+    def __repr__(self):
+        return '%s(%s)' % (self.__class__.__name__, self._wrapped_name)

jedi/evaluate/param.py

@@ -1,254 +1,111 @@
 from collections import defaultdict
 
-from jedi._compatibility import zip_longest
 from jedi import debug
-from jedi import common
-from jedi.parser import tree
-from jedi.evaluate import iterable
+from jedi.evaluate.utils import PushBackIterator
 from jedi.evaluate import analysis
-from jedi.evaluate import context
+from jedi.evaluate.lazy_context import LazyKnownContext, \
+    LazyTreeContext, LazyUnknownContext
 from jedi.evaluate import docstrings
-from jedi.evaluate import pep0484
-from jedi.evaluate.filters import ParamName
+from jedi.evaluate.context import iterable
 
 
-def add_argument_issue(parent_context, error_name, lazy_context, message):
-    if isinstance(lazy_context, context.LazyTreeContext):
+def _add_argument_issue(error_name, lazy_context, message):
+    if isinstance(lazy_context, LazyTreeContext):
         node = lazy_context.data
         if node.parent.type == 'argument':
             node = node.parent
-        analysis.add(parent_context, error_name, node, message)
-
-
-def try_iter_content(types, depth=0):
-    """Helper method for static analysis."""
-    if depth > 10:
-        # It's possible that a loop has references on itself (especially with
-        # CompiledObject). Therefore don't loop infinitely.
-        return
-
-    for typ in types:
-        try:
-            f = typ.py__iter__
-        except AttributeError:
-            pass
-        else:
-            for lazy_context in f():
-                try_iter_content(lazy_context.infer(), depth + 1)
-
-
-class AbstractArguments():
-    context = None
-
-    def eval_argument_clinic(self, parameters):
-        """Uses a list with argument clinic information (see PEP 436)."""
-        iterator = self.unpack()
-        for i, (name, optional, allow_kwargs) in enumerate(parameters):
-            key, argument = next(iterator, (None, None))
-            if key is not None:
-                raise NotImplementedError
-            if argument is None and not optional:
-                debug.warning('TypeError: %s expected at least %s arguments, got %s',
-                              name, len(parameters), i)
-                raise ValueError
-            values = set() if argument is None else argument.infer()
-
-            if not values and not optional:
-                # For the stdlib we always want values. If we don't get them,
-                # that's ok, maybe something is too hard to resolve, however,
-                # we will not proceed with the evaluation of that function.
-                debug.warning('argument_clinic "%s" not resolvable.', name)
-                raise ValueError
-            yield values
-
-    def eval_all(self, func=None):
-        """
-        Evaluates all arguments as a support for static analysis
-        (normally Jedi).
-        """
-        for key, lazy_context in self.unpack():
-            types = lazy_context.infer()
-            try_iter_content(types)
-
-
-class TreeArguments(AbstractArguments):
-    def __init__(self, evaluator, context, argument_node, trailer=None):
-        """
-        The argument_node is either a parser node or a list of evaluated
-        objects. Those evaluated objects may be lists of evaluated objects
-        themselves (one list for the first argument, one for the second, etc).
-
-        :param argument_node: May be an argument_node or a list of nodes.
-        """
-        self.argument_node = argument_node
-        self.context = context
-        self._evaluator = evaluator
-        self.trailer = trailer  # Can be None, e.g. in a class definition.
-
-    def _split(self):
-        if isinstance(self.argument_node, (tuple, list)):
-            for el in self.argument_node:
-                yield 0, el
-        else:
-            if not (self.argument_node.type == 'arglist' or (
-                    # in python 3.5 **arg is an argument, not arglist
-                    (self.argument_node.type == 'argument') and
-                     self.argument_node.children[0] in ('*', '**'))):
-                yield 0, self.argument_node
-                return
-
-            iterator = iter(self.argument_node.children)
-            for child in iterator:
-                if child == ',':
-                    continue
-                elif child in ('*', '**'):
-                    yield len(child.value), next(iterator)
-                elif child.type == 'argument' and \
-                        child.children[0] in ('*', '**'):
-                    assert len(child.children) == 2
-                    yield len(child.children[0].value), child.children[1]
-                else:
-                    yield 0, child
-
-    def unpack(self, func=None):
-        named_args = []
-        for stars, el in self._split():
-            if stars == 1:
-                arrays = self.context.eval_node(el)
-                iterators = [_iterate_star_args(self.context, a, el, func)
-                             for a in arrays]
-                iterators = list(iterators)
-                for values in list(zip_longest(*iterators)):
-                    # TODO zip_longest yields None, that means this would raise
-                    # an exception?
-                    yield None, context.get_merged_lazy_context(
-                        [v for v in values if v is not None]
-                    )
-            elif stars == 2:
-                arrays = self._evaluator.eval_element(self.context, el)
-                for dct in arrays:
-                    for key, values in _star_star_dict(self.context, dct, el, func):
-                        yield key, values
-            else:
-                if el.type == 'argument':
-                    c = el.children
-                    if len(c) == 3:  # Keyword argument.
-                        named_args.append((c[0].value, context.LazyTreeContext(self.context, c[2]),))
-                    else:  # Generator comprehension.
-                        # Include the brackets with the parent.
-                        comp = iterable.GeneratorComprehension(
-                            self._evaluator, self.context, self.argument_node.parent)
-                        yield None, context.LazyKnownContext(comp)
-                else:
-                    yield None, context.LazyTreeContext(self.context, el)
-
-        # Reordering var_args is necessary, because star args sometimes appear
-        # after named argument, but in the actual order it's prepended.
-        for named_arg in named_args:
-            yield named_arg
-
-    def as_tree_tuple_objects(self):
-        for stars, argument in self._split():
-            if argument.type == 'argument':
-                argument, default = argument.children[::2]
-            else:
-                default = None
-            yield argument, default, stars
-
-    def __repr__(self):
-        return '<%s: %s>' % (self.__class__.__name__, self.argument_node)
-
-    def get_calling_nodes(self):
-        from jedi.evaluate.dynamic import MergedExecutedParams
-        old_arguments_list = []
-        arguments = self
-
-        while arguments not in old_arguments_list:
-            if not isinstance(arguments, TreeArguments):
-                break
-
-            old_arguments_list.append(arguments)
-            for name, default, stars in reversed(list(arguments.as_tree_tuple_objects())):
-                if not stars or not isinstance(name, tree.Name):
-                    continue
-
-                names = self._evaluator.goto(arguments.context, name)
-                if len(names) != 1:
-                    break
-                if not isinstance(names[0], ParamName):
-                    break
-                param = names[0].get_param()
-                if isinstance(param, MergedExecutedParams):
-                    # For dynamic searches we don't even want to see errors.
-                    return []
-                if not isinstance(param, ExecutedParam):
-                    break
-                if param.var_args is None:
-                    break
-                arguments = param.var_args
-                break
-
-        return [arguments.argument_node or arguments.trailer]
-
-
-class ValuesArguments(AbstractArguments):
-    def __init__(self, values_list):
-        self._values_list = values_list
-
-    def unpack(self, func=None):
-        for values in self._values_list:
-            yield None, context.LazyKnownContexts(values)
-
-    def get_calling_nodes(self):
-        return []
-
-    def __repr__(self):
-        return '<%s: %s>' % (self.__class__.__name__, self._values_list)
+        return analysis.add(lazy_context.context, error_name, node, message)
 
 
 class ExecutedParam(object):
     """Fake a param and give it values."""
-    def __init__(self, var_args_context, original_param, var_args, lazy_context):
-        self._root_context = var_args_context.get_root_context()
-        self._original_param = original_param
-        self.var_args = var_args
+    def __init__(self, execution_context, param_node, lazy_context, is_default=False):
+        self._execution_context = execution_context
+        self._param_node = param_node
         self._lazy_context = lazy_context
-        self.string_name = self._original_param.name.value
+        self.string_name = param_node.name.value
+        self._is_default = is_default
 
-    def infer(self):
-        pep0484_hints = pep0484.follow_param(self._root_context, self._original_param)
-        doc_params = docstrings.follow_param(self._root_context, self._original_param)
-        if pep0484_hints or doc_params:
-            return list(set(pep0484_hints) | set(doc_params))
+    def infer_annotations(self):
+        from jedi.evaluate.gradual.annotation import infer_param
+        return infer_param(self._execution_context, self._param_node)
+
+    def infer(self, use_hints=True):
+        if use_hints:
+            doc_params = docstrings.infer_param(self._execution_context, self._param_node)
+            ann = self.infer_annotations().execute_annotation()
+            if ann or doc_params:
+                return ann | doc_params
 
         return self._lazy_context.infer()
 
+    def matches_signature(self):
+        if self._is_default:
+            return True
+        argument_contexts = self.infer(use_hints=False).py__class__()
+        if self._param_node.star_count:
+            return True
+        annotations = self.infer_annotations()
+        if not annotations:
+            # If we cannot infer annotations - or there aren't any - pretend
+            # that the signature matches.
+            return True
+        matches = any(c1.is_sub_class_of(c2)
+                      for c1 in argument_contexts
+                      for c2 in annotations.gather_annotation_classes())
+        debug.dbg("signature compare %s: %s <=> %s",
+                  matches, argument_contexts, annotations, color='BLUE')
+        return matches
+
     @property
-    def position_nr(self):
-        # Need to use the original logic here, because it uses the parent.
-        return self._original_param.position_nr
+    def var_args(self):
+        return self._execution_context.var_args
 
     def __repr__(self):
         return '<%s: %s>' % (self.__class__.__name__, self.string_name)
 
 
-def get_params(evaluator, parent_context, func, var_args):
+def get_executed_params_and_issues(execution_context, arguments):
+    def too_many_args(argument):
+        m = _error_argument_count(funcdef, len(unpacked_va))
+        # Just report an error for the first param that is not needed (like
+        # cPython).
+        if arguments.get_calling_nodes():
+            # There might not be a valid calling node so check for that first.
+            issues.append(
+                _add_argument_issue(
+                    'type-error-too-many-arguments',
+                    argument,
+                    message=m
+                )
+            )
+        else:
+            issues.append(None)
+
+    issues = []  # List[Optional[analysis issue]]
     result_params = []
     param_dict = {}
-    for param in func.params:
-        param_dict[str(param.name)] = param
-    unpacked_va = list(var_args.unpack(func))
-    var_arg_iterator = common.PushBackIterator(iter(unpacked_va))
+    funcdef = execution_context.tree_node
+    # Default params are part of the context where the function was defined.
+    # This means that they might have access on class variables that the
+    # function itself doesn't have.
+    default_param_context = execution_context.function_context.get_default_param_context()
+
+    for param in funcdef.get_params():
+        param_dict[param.name.value] = param
+    unpacked_va = list(arguments.unpack(funcdef))
+    var_arg_iterator = PushBackIterator(iter(unpacked_va))
 
     non_matching_keys = defaultdict(lambda: [])
     keys_used = {}
     keys_only = False
     had_multiple_value_error = False
-    for param in func.params:
+    for param in funcdef.get_params():
         # The value and key can both be null. There, the defaults apply.
         # args / kwargs will just be empty arrays / dicts, respectively.
         # Wrong value count is just ignored. If you try to test cases that are
         # not allowed in Python, Jedi will maybe not show any completions.
+        is_default = False
         key, argument = next(var_arg_iterator, (None, None))
         while key is not None:
             keys_only = True
@@ -260,12 +117,15 @@ def get_params(evaluator, parent_context, func, var_args):
                 if key in keys_used:
                     had_multiple_value_error = True
                     m = ("TypeError: %s() got multiple values for keyword argument '%s'."
-                         % (func.name, key))
-                    for node in var_args.get_calling_nodes():
-                        analysis.add(parent_context, 'type-error-multiple-values',
-                                     node, message=m)
+                         % (funcdef.name, key))
+                    for contextualized_node in arguments.get_calling_nodes():
+                        issues.append(
+                            analysis.add(contextualized_node.context,
+                                         'type-error-multiple-values',
+                                         contextualized_node.node, message=m)
+                        )
                 else:
-                    keys_used[key] = ExecutedParam(parent_context, key_param, var_args, argument)
+                    keys_used[key] = ExecutedParam(execution_context, key_param, argument)
             key, argument = next(var_arg_iterator, (None, None))
 
         try:
@@ -274,7 +134,7 @@ def get_params(evaluator, parent_context, func, var_args):
         except KeyError:
             pass
 
-        if param.stars == 1:
+        if param.star_count == 1:
             # *args param
             lazy_context_list = []
             if argument is not None:
@@ -285,31 +145,43 @@ def get_params(evaluator, parent_context, func, var_args):
                         var_arg_iterator.push_back((key, argument))
                         break
                     lazy_context_list.append(argument)
-            seq = iterable.FakeSequence(evaluator, 'tuple', lazy_context_list)
-            result_arg = context.LazyKnownContext(seq)
-        elif param.stars == 2:
+            seq = iterable.FakeSequence(execution_context.evaluator, u'tuple', lazy_context_list)
+            result_arg = LazyKnownContext(seq)
+        elif param.star_count == 2:
+            if argument is not None:
+                too_many_args(argument)
             # **kwargs param
-            dct = iterable.FakeDict(evaluator, dict(non_matching_keys))
-            result_arg = context.LazyKnownContext(dct)
+            dct = iterable.FakeDict(execution_context.evaluator, dict(non_matching_keys))
+            result_arg = LazyKnownContext(dct)
             non_matching_keys = {}
         else:
             # normal param
             if argument is None:
                 # No value: Return an empty container
                 if param.default is None:
-                    result_arg = context.LazyUnknownContext()
+                    result_arg = LazyUnknownContext()
                     if not keys_only:
-                        for node in var_args.get_calling_nodes():
-                            m = _error_argument_count(func, len(unpacked_va))
-                            analysis.add(parent_context, 'type-error-too-few-arguments',
-                                         node, message=m)
+                        for contextualized_node in arguments.get_calling_nodes():
+                            m = _error_argument_count(funcdef, len(unpacked_va))
+                            issues.append(
+                                analysis.add(
+                                    contextualized_node.context,
+                                    'type-error-too-few-arguments',
+                                    contextualized_node.node,
+                                    message=m,
+                                )
+                            )
                 else:
-                    result_arg = context.LazyTreeContext(parent_context, param.default)
+                    result_arg = LazyTreeContext(default_param_context, param.default)
+                    is_default = True
             else:
                 result_arg = argument
 
-        result_params.append(ExecutedParam(parent_context, param, var_args, result_arg))
-        if not isinstance(result_arg, context.LazyUnknownContext):
+        result_params.append(ExecutedParam(
+            execution_context, param, result_arg,
+            is_default=is_default
+        ))
+        if not isinstance(result_arg, LazyUnknownContext):
             keys_used[param.name.value] = result_params[-1]
 
     if keys_only:
@@ -320,87 +192,62 @@ def get_params(evaluator, parent_context, func, var_args):
             param = param_dict[k]
 
             if not (non_matching_keys or had_multiple_value_error or
-                    param.stars or param.default):
+                    param.star_count or param.default):
                 # add a warning only if there's not another one.
-                for node in var_args.get_calling_nodes():
-                    m = _error_argument_count(func, len(unpacked_va))
-                    analysis.add(parent_context, 'type-error-too-few-arguments',
-                                 node, message=m)
+                for contextualized_node in arguments.get_calling_nodes():
+                    m = _error_argument_count(funcdef, len(unpacked_va))
+                    issues.append(
+                        analysis.add(contextualized_node.context,
+                                     'type-error-too-few-arguments',
+                                     contextualized_node.node, message=m)
+                    )
 
     for key, lazy_context in non_matching_keys.items():
         m = "TypeError: %s() got an unexpected keyword argument '%s'." \
-            % (func.name, key)
-        add_argument_issue(
-            parent_context,
-            'type-error-keyword-argument',
-            lazy_context,
-            message=m
+            % (funcdef.name, key)
+        issues.append(
+            _add_argument_issue(
+                'type-error-keyword-argument',
+                lazy_context,
+                message=m
+            )
         )
 
     remaining_arguments = list(var_arg_iterator)
     if remaining_arguments:
-        m = _error_argument_count(func, len(unpacked_va))
-        # Just report an error for the first param that is not needed (like
-        # cPython).
         first_key, lazy_context = remaining_arguments[0]
-        if var_args.get_calling_nodes():
-            # There might not be a valid calling node so check for that first.
-            add_argument_issue(parent_context, 'type-error-too-many-arguments', lazy_context, message=m)
-    return result_params
+        too_many_args(lazy_context)
+    return result_params, issues
 
 
-def _iterate_star_args(context, array, input_node, func=None):
-    try:
-        iter_ = array.py__iter__
-    except AttributeError:
-        if func is not None:
-            # TODO this func should not be needed.
-            m = "TypeError: %s() argument after * must be a sequence, not %s" \
-                % (func.name.value, array)
-            analysis.add(context, 'type-error-star', input_node, message=m)
-    else:
-        for lazy_context in iter_():
-            yield lazy_context
-
-
-def _star_star_dict(context, array, input_node, func):
-    from jedi.evaluate.instance import CompiledInstance
-    if isinstance(array, CompiledInstance) and array.name.string_name == 'dict':
-        # For now ignore this case. In the future add proper iterators and just
-        # make one call without crazy isinstance checks.
-        return {}
-    elif isinstance(array, iterable.AbstractSequence) and array.array_type == 'dict':
-        return array.exact_key_items()
-    else:
-        if func is not None:
-            m = "TypeError: %s argument after ** must be a mapping, not %s" \
-                % (func.name.value, array)
-            analysis.add(context, 'type-error-star-star', input_node, message=m)
-        return {}
-
-
-def _error_argument_count(func, actual_count):
-    default_arguments = sum(1 for p in func.params if p.default or p.stars)
+def _error_argument_count(funcdef, actual_count):
+    params = funcdef.get_params()
+    default_arguments = sum(1 for p in params if p.default or p.star_count)
 
     if default_arguments == 0:
         before = 'exactly '
     else:
-        before = 'from %s to ' % (len(func.params) - default_arguments)
+        before = 'from %s to ' % (len(params) - default_arguments)
     return ('TypeError: %s() takes %s%s arguments (%s given).'
-            % (func.name, before, len(func.params), actual_count))
+            % (funcdef.name, before, len(params), actual_count))
 
 
-def create_default_param(parent_context, param):
-    if param.stars == 1:
-        result_arg = context.LazyKnownContext(
-            iterable.FakeSequence(parent_context.evaluator, 'tuple', [])
+def _create_default_param(execution_context, param):
+    if param.star_count == 1:
+        result_arg = LazyKnownContext(
+            iterable.FakeSequence(execution_context.evaluator, u'tuple', [])
         )
-    elif param.stars == 2:
-        result_arg = context.LazyKnownContext(
-            iterable.FakeDict(parent_context.evaluator, {})
+    elif param.star_count == 2:
+        result_arg = LazyKnownContext(
+            iterable.FakeDict(execution_context.evaluator, {})
         )
     elif param.default is None:
-        result_arg = context.LazyUnknownContext()
+        result_arg = LazyUnknownContext()
     else:
-        result_arg = context.LazyTreeContext(parent_context, param.default)
-    return ExecutedParam(parent_context, param, None, result_arg)
+        result_arg = LazyTreeContext(execution_context.parent_context, param.default)
+    return ExecutedParam(execution_context, param, result_arg)
+
+
+def create_default_params(execution_context, funcdef):
+    return [_create_default_param(execution_context, p)
+            for p in funcdef.get_params()]

jedi/evaluate/parser_cache.py

@@ -0,0 +1,6 @@
+from jedi.evaluate.cache import evaluator_function_cache
+
+
+@evaluator_function_cache()
+def get_yield_exprs(evaluator, funcdef):
+    return list(funcdef.iter_yield_exprs())

jedi/evaluate/pep0484.py

@@ -1,215 +0,0 @@
-"""
-PEP 0484 ( https://www.python.org/dev/peps/pep-0484/ ) describes type hints
-through function annotations. There is a strong suggestion in this document
-that only the type of type hinting defined in PEP0484 should be allowed
-as annotations in future python versions.
-
-The (initial / probably incomplete) implementation todo list for pep-0484:
-v Function parameter annotations with builtin/custom type classes
-v Function returntype annotations with builtin/custom type classes
-v Function parameter annotations with strings (forward reference)
-v Function return type annotations with strings (forward reference)
-v Local variable type hints
-v Assigned types: `Url = str\ndef get(url:Url) -> str:`
-v Type hints in `with` statements
-x Stub files support
-x support `@no_type_check` and `@no_type_check_decorator`
-x support for typing.cast() operator
-x support for type hint comments for functions, `# type: (int, str) -> int`.
-    See comment from Guido https://github.com/davidhalter/jedi/issues/662
-"""
-
-import itertools
-
-import os
-from jedi.parser import \
-    Parser, load_grammar, ParseError, ParserWithRecovery, tree
-from jedi.common import unite
-from jedi.evaluate.cache import memoize_default
-from jedi.evaluate import compiled
-from jedi.evaluate.context import LazyTreeContext
-from jedi import debug
-from jedi import _compatibility
-import re
-
-
-def _evaluate_for_annotation(context, annotation, index=None):
-    """
-    Evaluates a string-node, looking for an annotation
-    If index is not None, the annotation is expected to be a tuple
-    and we're interested in that index
-    """
-    if annotation is not None:
-        definitions = context.eval_node(
-            _fix_forward_reference(context, annotation))
-        if index is not None:
-            definitions = list(itertools.chain.from_iterable(
-                definition.py__getitem__(index) for definition in definitions
-                if definition.array_type == 'tuple' and
-                len(list(definition.py__iter__())) >= index))
-        return unite(d.execute_evaluated() for d in definitions)
-    else:
-        return set()
-
-
-def _fix_forward_reference(context, node):
-    evaled_nodes = context.eval_node(node)
-    if len(evaled_nodes) != 1:
-        debug.warning("Eval'ed typing index %s should lead to 1 object, "
-                      " not %s" % (node, evaled_nodes))
-        return node
-    evaled_node = list(evaled_nodes)[0]
-    if isinstance(evaled_node, compiled.CompiledObject) and \
-            isinstance(evaled_node.obj, str):
-        try:
-            p = Parser(load_grammar(), _compatibility.unicode(evaled_node.obj),
-                       start_symbol='eval_input')
-            new_node = p.get_parsed_node()
-        except ParseError:
-            debug.warning('Annotation not parsed: %s' % evaled_node.obj)
-            return node
-        else:
-            module = node.get_parent_until()
-            new_node.move(module.end_pos[0])
-            new_node.parent = context.tree_node
-            return new_node
-    else:
-        return node
-
-
-@memoize_default()
-def follow_param(context, param):
-    annotation = param.annotation()
-    return _evaluate_for_annotation(context, annotation)
-
-
-def py__annotations__(funcdef):
-    return_annotation = funcdef.annotation()
-    if return_annotation:
-        dct = {'return': return_annotation}
-    else:
-        dct = {}
-    for function_param in funcdef.params:
-        param_annotation = function_param.annotation()
-        if param_annotation is not None:
-            dct[function_param.name.value] = param_annotation
-    return dct
-
-
-@memoize_default()
-def find_return_types(context, func):
-    annotation = py__annotations__(func).get("return", None)
-    return _evaluate_for_annotation(context, annotation)
-
-
-_typing_module = None
-
-
-def _get_typing_replacement_module():
-    """
-    The idea is to return our jedi replacement for the PEP-0484 typing module
-    as discussed at https://github.com/davidhalter/jedi/issues/663
-    """
-    global _typing_module
-    if _typing_module is None:
-        typing_path = \
-            os.path.abspath(os.path.join(__file__, "../jedi_typing.py"))
-        with open(typing_path) as f:
-            code = _compatibility.unicode(f.read())
-        p = ParserWithRecovery(load_grammar(), code)
-        _typing_module = p.module
-    return _typing_module
-
-
-def py__getitem__(context, typ, node):
-    if not typ.get_root_context().name.string_name == "typing":
-        return None
-    # we assume that any class using [] in a module called
-    # "typing" with a name for which we have a replacement
-    # should be replaced by that class. This is not 100%
-    # airtight but I don't have a better idea to check that it's
-    # actually the PEP-0484 typing module and not some other
-    if node.type == "subscriptlist":
-        nodes = node.children[::2]  # skip the commas
-    else:
-        nodes = [node]
-    del node
-
-    nodes = [_fix_forward_reference(context, node) for node in nodes]
-    type_name = typ.name.string_name
-
-    # hacked in Union and Optional, since it's hard to do nicely in parsed code
-    if type_name in ("Union", '_Union'):
-        # In Python 3.6 it's still called typing.Union but it's an instance
-        # called _Union.
-        return unite(context.eval_node(node) for node in nodes)
-    if type_name in ("Optional", '_Optional'):
-        # Here we have the same issue like in Union. Therefore we also need to
-        # check for the instance typing._Optional (Python 3.6).
-        return context.eval_node(nodes[0])
-
-    from jedi.evaluate.representation import ModuleContext
-    typing = ModuleContext(context.evaluator, _get_typing_replacement_module())
-    factories = typing.py__getattribute__("factory")
-    assert len(factories) == 1
-    factory = list(factories)[0]
-    assert factory
-    function_body_nodes = factory.tree_node.children[4].children
-    valid_classnames = set(child.name.value
-                           for child in function_body_nodes
-                           if isinstance(child, tree.Class))
-    if type_name not in valid_classnames:
-        return None
-    compiled_classname = compiled.create(context.evaluator, type_name)
-
-    from jedi.evaluate.iterable import FakeSequence
-    args = FakeSequence(
-        context.evaluator,
-        "tuple",
-        [LazyTreeContext(context, n) for n in nodes]
-    )
-
-    result = factory.execute_evaluated(compiled_classname, args)
-    return result
-
-
-def find_type_from_comment_hint_for(context, node, name):
-    return _find_type_from_comment_hint(context, node, node.children[1], name)
-
-
-def find_type_from_comment_hint_with(context, node, name):
-    assert len(node.children[1].children) == 3, \
-        "Can only be here when children[1] is 'foo() as f'"
-    varlist = node.children[1].children[2]
-    return _find_type_from_comment_hint(context, node, varlist, name)
-
-
-def find_type_from_comment_hint_assign(context, node, name):
-    return _find_type_from_comment_hint(context, node, node.children[0], name)
-
-
-def _find_type_from_comment_hint(context, node, varlist, name):
-    index = None
-    if varlist.type in ("testlist_star_expr", "exprlist"):
-        # something like "a, b = 1, 2"
-        index = 0
-        for child in varlist.children:
-            if child == name:
-                break
-            if child.type == "operator":
-                continue
-            index += 1
-        else:
-            return []
-
-    comment = node.get_following_comment_same_line()
-    if comment is None:
-        return []
-    match = re.match(r"^#\s*type:\s*([^#]*)", comment)
-    if not match:
-        return []
-    annotation = tree.String(
-        repr(str(match.group(1).strip())),
-        node.start_pos)
-    annotation.parent = node.parent
-    return _evaluate_for_annotation(context, annotation, index)

jedi/evaluate/precedence.py

@@ -1,179 +0,0 @@
-"""
-Handles operator precedence.
-"""
-import operator as op
-
-from jedi._compatibility import unicode
-from jedi.parser import tree
-from jedi import debug
-from jedi.evaluate.compiled import CompiledObject, create, builtin_from_name
-from jedi.evaluate import analysis
-
-# Maps Python syntax to the operator module.
-COMPARISON_OPERATORS = {
-    '==': op.eq,
-    '!=': op.ne,
-    'is': op.is_,
-    'is not': op.is_not,
-    '<': op.lt,
-    '<=': op.le,
-    '>': op.gt,
-    '>=': op.ge,
-}
-
-
-def literals_to_types(evaluator, result):
-    # Changes literals ('a', 1, 1.0, etc) to its type instances (str(),
-    # int(), float(), etc).
-    new_result = set()
-    for typ in result:
-        if is_literal(typ):
-            # Literals are only valid as long as the operations are
-            # correct. Otherwise add a value-free instance.
-            cls = builtin_from_name(evaluator, typ.name.string_name)
-            new_result |= cls.execute_evaluated()
-        else:
-            new_result.add(typ)
-    return new_result
-
-
-def calculate_children(evaluator, context, children):
-    """
-    Calculate a list of children with operators.
-    """
-    iterator = iter(children)
-    types = context.eval_node(next(iterator))
-    for operator in iterator:
-        right = next(iterator)
-        if operator.type == 'comp_op':  # not in / is not
-            operator = ' '.join(str(c.value) for c in operator.children)
-
-        # handle lazy evaluation of and/or here.
-        if operator in ('and', 'or'):
-            left_bools = set([left.py__bool__() for left in types])
-            if left_bools == set([True]):
-                if operator == 'and':
-                    types = context.eval_node(right)
-            elif left_bools == set([False]):
-                if operator != 'and':
-                    types = context.eval_node(right)
-            # Otherwise continue, because of uncertainty.
-        else:
-            types = calculate(evaluator, context, types, operator,
-                              context.eval_node(right))
-    debug.dbg('calculate_children types %s', types)
-    return types
-
-
-def calculate(evaluator, context, left_result, operator, right_result):
-    result = set()
-    if not left_result or not right_result:
-        # illegal slices e.g. cause left/right_result to be None
-        result = (left_result or set()) | (right_result or set())
-        result = literals_to_types(evaluator, result)
-    else:
-        # I don't think there's a reasonable chance that a string
-        # operation is still correct, once we pass something like six
-        # objects.
-        if len(left_result) * len(right_result) > 6:
-            result = literals_to_types(evaluator, left_result | right_result)
-        else:
-            for left in left_result:
-                for right in right_result:
-                    result |= _element_calculate(evaluator, context, left, operator, right)
-    return result
-
-
-def factor_calculate(evaluator, types, operator):
-    """
-    Calculates `+`, `-`, `~` and `not` prefixes.
-    """
-    for typ in types:
-        if operator == '-':
-            if _is_number(typ):
-                yield create(evaluator, -typ.obj)
-        elif operator == 'not':
-            value = typ.py__bool__()
-            if value is None:  # Uncertainty.
-                return
-            yield create(evaluator, not value)
-        else:
-            yield typ
-
-
-def _is_number(obj):
-    return isinstance(obj, CompiledObject) \
-        and isinstance(obj.obj, (int, float))
-
-
-def is_string(obj):
-    return isinstance(obj, CompiledObject) \
-        and isinstance(obj.obj, (str, unicode))
-
-
-def is_literal(obj):
-    return _is_number(obj) or is_string(obj)
-
-
-def _is_tuple(obj):
-    from jedi.evaluate import iterable
-    return isinstance(obj, iterable.AbstractSequence) and obj.array_type == 'tuple'
-
-
-def _is_list(obj):
-    from jedi.evaluate import iterable
-    return isinstance(obj, iterable.AbstractSequence) and obj.array_type == 'list'
-
-
-def _element_calculate(evaluator, context, left, operator, right):
-    from jedi.evaluate import iterable, instance
-    l_is_num = _is_number(left)
-    r_is_num = _is_number(right)
-    if operator == '*':
-        # for iterables, ignore * operations
-        if isinstance(left, iterable.AbstractSequence) or is_string(left):
-            return set([left])
-        elif isinstance(right, iterable.AbstractSequence) or is_string(right):
-            return set([right])
-    elif operator == '+':
-        if l_is_num and r_is_num or is_string(left) and is_string(right):
-            return set([create(evaluator, left.obj + right.obj)])
-        elif _is_tuple(left) and _is_tuple(right) or _is_list(left) and _is_list(right):
-            return set([iterable.MergedArray(evaluator, (left, right))])
-    elif operator == '-':
-        if l_is_num and r_is_num:
-            return set([create(evaluator, left.obj - right.obj)])
-    elif operator == '%':
-        # With strings and numbers the left type typically remains. Except for
-        # `int() % float()`.
-        return set([left])
-    elif operator in COMPARISON_OPERATORS:
-        operation = COMPARISON_OPERATORS[operator]
-        if isinstance(left, CompiledObject) and isinstance(right, CompiledObject):
-            # Possible, because the return is not an option. Just compare.
-            left = left.obj
-            right = right.obj
-
-        try:
-            result = operation(left, right)
-        except TypeError:
-            # Could be True or False.
-            return set([create(evaluator, True), create(evaluator, False)])
-        else:
-            return set([create(evaluator, result)])
-    elif operator == 'in':
-        return set()
-
-    def check(obj):
-        """Checks if a Jedi object is either a float or an int."""
-        return isinstance(obj, instance.CompiledInstance) and \
-            obj.name.string_name in ('int', 'float')
-
-    # Static analysis, one is a number, the other one is not.
-    if operator in ('+', '-') and l_is_num != r_is_num \
-            and not (check(left) or check(right)):
-        message = "TypeError: unsupported operand type(s) for +: %s and %s"
-        analysis.add(context, 'type-error-operation', operator,
-                     message % (left, right))
-
-    return set([left, right])

jedi/evaluate/recursion.py

@@ -6,11 +6,48 @@ the right time. You can read more about them :ref:`here <settings-recursion>`.
 Next to :mod:`jedi.evaluate.cache` this module also makes |jedi| not
 thread-safe. Why?  ``execution_recursion_decorator`` uses class variables to
 count the function calls.
+
+.. _settings-recursion:
+
+Settings
+~~~~~~~~~~
+
+Recursion settings are important if you don't want extremly
+recursive python code to go absolutely crazy.
+
+The default values are based on experiments while completing the |jedi| library
+itself (inception!). But I don't think there's any other Python library that
+uses recursion in a similarly extreme way. Completion should also be fast and
+therefore the quality might not always be maximal.
+
+.. autodata:: recursion_limit
+.. autodata:: total_function_execution_limit
+.. autodata:: per_function_execution_limit
+.. autodata:: per_function_recursion_limit
 """
+
 from contextlib import contextmanager
 
 from jedi import debug
-from jedi import settings
+from jedi.evaluate.base_context import NO_CONTEXTS
+
+
+recursion_limit = 15
+"""
+Like ``sys.getrecursionlimit()``, just for |jedi|.
+"""
+total_function_execution_limit = 200
+"""
+This is a hard limit of how many non-builtin functions can be executed.
+"""
+per_function_execution_limit = 6
+"""
+The maximal amount of times a specific function may be executed.
+"""
+per_function_recursion_limit = 2
+"""
+A function may not be executed more than this number of times recursively.
+"""
 
 
 class RecursionDetector(object):
@@ -28,24 +65,26 @@ def execution_allowed(evaluator, node):
 
     if node in pushed_nodes:
         debug.warning('catched stmt recursion: %s @%s', node,
-                      node.start_pos)
+                      getattr(node, 'start_pos', None))
         yield False
     else:
-        pushed_nodes.append(node)
-        yield True
-        pushed_nodes.pop()
+        try:
+            pushed_nodes.append(node)
+            yield True
+        finally:
+            pushed_nodes.pop()
 
 
-def execution_recursion_decorator(default=set()):
+def execution_recursion_decorator(default=NO_CONTEXTS):
     def decorator(func):
-        def wrapper(execution, **kwargs):
-            detector = execution.evaluator.execution_recursion_detector
-            allowed = detector.push_execution(execution)
+        def wrapper(self, **kwargs):
+            detector = self.evaluator.execution_recursion_detector
+            limit_reached = detector.push_execution(self)
             try:
-                if allowed:
+                if limit_reached:
                     result = default
                 else:
-                    result = func(execution, **kwargs)
+                    result = func(self, **kwargs)
             finally:
                 detector.pop_execution()
             return result
@@ -58,47 +97,57 @@ class ExecutionRecursionDetector(object):
     Catches recursions of executions.
     """
     def __init__(self, evaluator):
-        self.recursion_level = 0
-        self.parent_execution_funcs = []
-        self.execution_funcs = set()
-        self.execution_count = 0
         self._evaluator = evaluator
 
-    def __call__(self, execution):
-        debug.dbg('Execution recursions: %s', execution, self.recursion_level,
-                  self.execution_count, len(self.execution_funcs))
-        if self.check_recursion(execution):
-            result = set()
-        else:
-            result = self.func(execution)
-        self.pop_execution()
-        return result
+        self._recursion_level = 0
+        self._parent_execution_funcs = []
+        self._funcdef_execution_counts = {}
+        self._execution_count = 0
 
     def pop_execution(self):
-        self.parent_execution_funcs.pop()
-        self.recursion_level -= 1
+        self._parent_execution_funcs.pop()
+        self._recursion_level -= 1
 
     def push_execution(self, execution):
-        in_par_execution_funcs = execution.tree_node in self.parent_execution_funcs
-        in_execution_funcs = execution.tree_node in self.execution_funcs
-        self.recursion_level += 1
-        self.execution_count += 1
-        self.execution_funcs.add(execution.tree_node)
-        self.parent_execution_funcs.append(execution.tree_node)
+        funcdef = execution.tree_node
 
-        if self.execution_count > settings.max_executions:
-            return True
+        # These two will be undone in pop_execution.
+        self._recursion_level += 1
+        self._parent_execution_funcs.append(funcdef)
 
         module = execution.get_root_context()
-        if module == self._evaluator.BUILTINS:
+
+        if module == self._evaluator.builtins_module:
+            # We have control over builtins so we know they are not recursing
+            # like crazy. Therefore we just let them execute always, because
+            # they usually just help a lot with getting good results.
             return False
 
-        if in_par_execution_funcs:
-            if self.recursion_level > settings.max_function_recursion_level:
-                return True
-        if in_execution_funcs and \
-                len(self.execution_funcs) > settings.max_until_execution_unique:
+        if self._recursion_level > recursion_limit:
+            debug.warning('Recursion limit (%s) reached', recursion_limit)
             return True
-        if self.execution_count > settings.max_executions_without_builtins:
+
+        if self._execution_count >= total_function_execution_limit:
+            debug.warning('Function execution limit (%s) reached', total_function_execution_limit)
+            return True
+        self._execution_count += 1
+
+        if self._funcdef_execution_counts.setdefault(funcdef, 0) >= per_function_execution_limit:
+            if module.py__name__() in ('builtins', 'typing'):
+                return False
+            debug.warning(
+                'Per function execution limit (%s) reached: %s',
+                per_function_execution_limit,
+                funcdef
+            )
+            return True
+        self._funcdef_execution_counts[funcdef] += 1
+
+        if self._parent_execution_funcs.count(funcdef) > per_function_recursion_limit:
+            debug.warning(
+                'Per function recursion limit (%s) reached: %s',
+                per_function_recursion_limit,
+                funcdef
+            )
             return True
         return False

jedi/evaluate/representation.py

@@ -1,559 +0,0 @@
-"""
-Like described in the :mod:`jedi.parser.tree` module,
-there's a need for an ast like module to represent the states of parsed
-modules.
-
-But now there are also structures in Python that need a little bit more than
-that. An ``Instance`` for example is only a ``Class`` before it is
-instantiated. This class represents these cases.
-
-So, why is there also a ``Class`` class here? Well, there are decorators and
-they change classes in Python 3.
-
-Representation modules also define "magic methods". Those methods look like
-``py__foo__`` and are typically mappable to the Python equivalents ``__call__``
-and others. Here's a list:
-
-====================================== ========================================
-**Method**                             **Description**
--------------------------------------- ----------------------------------------
-py__call__(params: Array)              On callable objects, returns types.
-py__bool__()                           Returns True/False/None; None means that
-                                       there's no certainty.
-py__bases__()                          Returns a list of base classes.
-py__mro__()                            Returns a list of classes (the mro).
-py__iter__()                           Returns a generator of a set of types.
-py__class__()                          Returns the class of an instance.
-py__getitem__(index: int/str)          Returns a a set of types of the index.
-                                       Can raise an IndexError/KeyError.
-py__file__()                           Only on modules. Returns None if does
-                                       not exist.
-py__package__()                        Only on modules. For the import system.
-py__path__()                           Only on modules. For the import system.
-py__get__(call_object)                 Only on instances. Simulates
-                                       descriptors.
-====================================== ========================================
-
-"""
-import os
-import pkgutil
-import imp
-import re
-
-from jedi._compatibility import use_metaclass
-from jedi.parser import tree
-from jedi import debug
-from jedi import common
-from jedi.evaluate.cache import memoize_default, CachedMetaClass, NO_DEFAULT
-from jedi.evaluate import compiled
-from jedi.evaluate import recursion
-from jedi.evaluate import iterable
-from jedi.evaluate import docstrings
-from jedi.evaluate import pep0484
-from jedi.evaluate import param
-from jedi.evaluate import flow_analysis
-from jedi.evaluate import imports
-from jedi.evaluate import helpers
-from jedi.evaluate.filters import ParserTreeFilter, FunctionExecutionFilter, \
-    GlobalNameFilter, DictFilter, ContextName, AbstractNameDefinition, \
-    ParamName, AnonymousInstanceParamName, TreeNameDefinition
-from jedi.evaluate.dynamic import search_params
-from jedi.evaluate import context
-
-
-def apply_py__get__(context, base_context):
-    try:
-        method = context.py__get__
-    except AttributeError:
-        yield context
-    else:
-        for descriptor_context in method(base_context):
-            yield descriptor_context
-
-
-class ClassName(TreeNameDefinition):
-    def infer(self):
-        for result_context in super(ClassName, self).infer():
-            for c in apply_py__get__(result_context, self.parent_context):
-                yield c
-
-
-class ClassFilter(ParserTreeFilter):
-    name_class = ClassName
-
-
-class ClassContext(use_metaclass(CachedMetaClass, context.TreeContext)):
-    """
-    This class is not only important to extend `tree.Class`, it is also a
-    important for descriptors (if the descriptor methods are evaluated or not).
-    """
-    api_type = 'class'
-
-    def __init__(self, evaluator, classdef, parent_context):
-        super(ClassContext, self).__init__(evaluator, parent_context=parent_context)
-        self.tree_node = classdef
-
-    @memoize_default(default=())
-    def py__mro__(self):
-        def add(cls):
-            if cls not in mro:
-                mro.append(cls)
-
-        mro = [self]
-        # TODO Do a proper mro resolution. Currently we are just listing
-        # classes. However, it's a complicated algorithm.
-        for lazy_cls in self.py__bases__():
-            # TODO there's multiple different mro paths possible if this yields
-            # multiple possibilities. Could be changed to be more correct.
-            for cls in lazy_cls.infer():
-                # TODO detect for TypeError: duplicate base class str,
-                # e.g.  `class X(str, str): pass`
-                try:
-                    mro_method = cls.py__mro__
-                except AttributeError:
-                    # TODO add a TypeError like:
-                    """
-                    >>> class Y(lambda: test): pass
-                    Traceback (most recent call last):
-                      File "<stdin>", line 1, in <module>
-                    TypeError: function() argument 1 must be code, not str
-                    >>> class Y(1): pass
-                    Traceback (most recent call last):
-                      File "<stdin>", line 1, in <module>
-                    TypeError: int() takes at most 2 arguments (3 given)
-                    """
-                    pass
-                else:
-                    add(cls)
-                    for cls_new in mro_method():
-                        add(cls_new)
-        return tuple(mro)
-
-    @memoize_default(default=())
-    def py__bases__(self):
-        arglist = self.tree_node.get_super_arglist()
-        if arglist:
-            args = param.TreeArguments(self.evaluator, self, arglist)
-            return [value for key, value in args.unpack() if key is None]
-        else:
-            return [context.LazyKnownContext(compiled.create(self.evaluator, object))]
-
-    def py__call__(self, params):
-        from jedi.evaluate.instance import TreeInstance
-        return set([TreeInstance(self.evaluator, self.parent_context, self, params)])
-
-    def py__class__(self):
-        return compiled.create(self.evaluator, type)
-
-    def get_params(self):
-        from jedi.evaluate.instance import AnonymousInstance
-        anon = AnonymousInstance(self.evaluator, self.parent_context, self)
-        return [AnonymousInstanceParamName(anon, param.name) for param in self.funcdef.params]
-
-    def get_filters(self, search_global, until_position=None, origin_scope=None, is_instance=False):
-        if search_global:
-            yield ParserTreeFilter(
-                self.evaluator,
-                context=self,
-                until_position=until_position,
-                origin_scope=origin_scope
-            )
-        else:
-            for scope in self.py__mro__():
-                if isinstance(scope, compiled.CompiledObject):
-                    for filter in scope.get_filters(is_instance=is_instance):
-                        yield filter
-                else:
-                    yield ClassFilter(
-                        self.evaluator, self, node_context=scope,
-                        origin_scope=origin_scope)
-
-    def is_class(self):
-        return True
-
-    def get_subscope_by_name(self, name):
-        raise DeprecationWarning
-        for s in self.py__mro__():
-            for sub in reversed(s.subscopes):
-                if sub.name.value == name:
-                    return sub
-        raise KeyError("Couldn't find subscope.")
-
-    def get_function_slot_names(self, name):
-        for filter in self.get_filters(search_global=False):
-            names = filter.get(name)
-            if names:
-                return names
-        return []
-
-    def get_param_names(self):
-        for name in self.get_function_slot_names('__init__'):
-            for context_ in name.infer():
-                try:
-                    method = context_.get_param_names
-                except AttributeError:
-                    pass
-                else:
-                    return list(method())[1:]
-        return []
-
-    @property
-    def name(self):
-        return ContextName(self, self.tree_node.name)
-
-
-class FunctionContext(use_metaclass(CachedMetaClass, context.TreeContext)):
-    """
-    Needed because of decorators. Decorators are evaluated here.
-    """
-    api_type = 'function'
-
-    def __init__(self, evaluator, parent_context, funcdef):
-        """ This should not be called directly """
-        super(FunctionContext, self).__init__(evaluator, parent_context)
-        self.tree_node = funcdef
-
-    def get_filters(self, search_global, until_position=None, origin_scope=None):
-        if search_global:
-            yield ParserTreeFilter(
-                self.evaluator,
-                context=self,
-                until_position=until_position,
-                origin_scope=origin_scope
-            )
-        else:
-            scope = self.py__class__()
-            for filter in scope.get_filters(search_global=False, origin_scope=origin_scope):
-                yield filter
-
-    def infer_function_execution(self, function_execution):
-        """
-        Created to be used by inheritance.
-        """
-        if self.tree_node.is_generator():
-            return set([iterable.Generator(self.evaluator, function_execution)])
-        else:
-            return function_execution.get_return_values()
-
-    def get_function_execution(self, arguments=None):
-        e = self.evaluator
-        if arguments is None:
-            return AnonymousFunctionExecution(e, self.parent_context, self)
-        else:
-            return FunctionExecutionContext(e, self.parent_context, self, arguments)
-
-    def py__call__(self, arguments):
-        function_execution = self.get_function_execution(arguments)
-        return self.infer_function_execution(function_execution)
-
-    def py__class__(self):
-        # This differentiation is only necessary for Python2. Python3 does not
-        # use a different method class.
-        if isinstance(self.tree_node.get_parent_scope(), tree.Class):
-            name = 'METHOD_CLASS'
-        else:
-            name = 'FUNCTION_CLASS'
-        return compiled.get_special_object(self.evaluator, name)
-
-    @property
-    def name(self):
-        return ContextName(self, self.tree_node.name)
-
-    def get_param_names(self):
-        function_execution = self.get_function_execution()
-        return [ParamName(function_execution, param.name) for param in self.tree_node.params]
-
-
-class FunctionExecutionContext(context.TreeContext):
-    """
-    This class is used to evaluate functions and their returns.
-
-    This is the most complicated class, because it contains the logic to
-    transfer parameters. It is even more complicated, because there may be
-    multiple calls to functions and recursion has to be avoided. But this is
-    responsibility of the decorators.
-    """
-    function_execution_filter = FunctionExecutionFilter
-
-    def __init__(self, evaluator, parent_context, function_context, var_args):
-        super(FunctionExecutionContext, self).__init__(evaluator, parent_context)
-        self.function_context = function_context
-        self.tree_node = function_context.tree_node
-        self.var_args = var_args
-
-    @memoize_default(default=set())
-    @recursion.execution_recursion_decorator()
-    def get_return_values(self, check_yields=False):
-        funcdef = self.tree_node
-        if funcdef.type == 'lambda':
-            return self.evaluator.eval_element(self, funcdef.children[-1])
-
-        if check_yields:
-            types = set()
-            returns = funcdef.yields
-        else:
-            returns = funcdef.returns
-            types = set(docstrings.find_return_types(self.get_root_context(), funcdef))
-            types |= set(pep0484.find_return_types(self.get_root_context(), funcdef))
-
-        for r in returns:
-            check = flow_analysis.reachability_check(self, funcdef, r)
-            if check is flow_analysis.UNREACHABLE:
-                debug.dbg('Return unreachable: %s', r)
-            else:
-                if check_yields:
-                    types |= set(self._eval_yield(r))
-                else:
-                    types |= self.eval_node(r.children[1])
-            if check is flow_analysis.REACHABLE:
-                debug.dbg('Return reachable: %s', r)
-                break
-        return types
-
-    def _eval_yield(self, yield_expr):
-        node = yield_expr.children[1]
-        if node.type == 'yield_arg':  # It must be a yield from.
-            yield_from_types = self.eval_node(node.children[1])
-            for lazy_context in iterable.py__iter__(self.evaluator, yield_from_types, node):
-                yield lazy_context
-        else:
-            yield context.LazyTreeContext(self, node)
-
-    @recursion.execution_recursion_decorator(default=iter([]))
-    def get_yield_values(self):
-        for_parents = [(y, tree.search_ancestor(y, ('for_stmt', 'funcdef',
-                                                    'while_stmt', 'if_stmt')))
-                       for y in self.tree_node.yields]
-
-        # Calculate if the yields are placed within the same for loop.
-        yields_order = []
-        last_for_stmt = None
-        for yield_, for_stmt in for_parents:
-            # For really simple for loops we can predict the order. Otherwise
-            # we just ignore it.
-            parent = for_stmt.parent
-            if parent.type == 'suite':
-                parent = parent.parent
-            if for_stmt.type == 'for_stmt' and parent == self.tree_node \
-                    and for_stmt.defines_one_name():  # Simplicity for now.
-                if for_stmt == last_for_stmt:
-                    yields_order[-1][1].append(yield_)
-                else:
-                    yields_order.append((for_stmt, [yield_]))
-            elif for_stmt == self.tree_node:
-                yields_order.append((None, [yield_]))
-            else:
-                types = self.get_return_values(check_yields=True)
-                if types:
-                    yield context.get_merged_lazy_context(list(types))
-                return
-            last_for_stmt = for_stmt
-
-        evaluator = self.evaluator
-        for for_stmt, yields in yields_order:
-            if for_stmt is None:
-                # No for_stmt, just normal yields.
-                for yield_ in yields:
-                    for result in self._eval_yield(yield_):
-                        yield result
-            else:
-                input_node = for_stmt.get_input_node()
-                for_types = self.eval_node(input_node)
-                ordered = iterable.py__iter__(evaluator, for_types, input_node)
-                ordered = list(ordered)
-                for lazy_context in ordered:
-                    dct = {str(for_stmt.children[1]): lazy_context.infer()}
-                    with helpers.predefine_names(self, for_stmt, dct):
-                        for yield_in_same_for_stmt in yields:
-                            for result in self._eval_yield(yield_in_same_for_stmt):
-                                yield result
-
-    def get_filters(self, search_global, until_position=None, origin_scope=None):
-        yield self.function_execution_filter(self.evaluator, self,
-                                             until_position=until_position,
-                                             origin_scope=origin_scope)
-
-    @memoize_default(default=NO_DEFAULT)
-    def get_params(self):
-        return param.get_params(self.evaluator, self.parent_context, self.tree_node, self.var_args)
-
-
-class AnonymousFunctionExecution(FunctionExecutionContext):
-    def __init__(self, evaluator, parent_context, function_context):
-        super(AnonymousFunctionExecution, self).__init__(
-            evaluator, parent_context, function_context, var_args=None)
-
-    @memoize_default(default=NO_DEFAULT)
-    def get_params(self):
-        # We need to do a dynamic search here.
-        return search_params(self.evaluator, self.parent_context, self.tree_node)
-
-
-class ModuleAttributeName(AbstractNameDefinition):
-    """
-    For module attributes like __file__, __str__ and so on.
-    """
-    api_type = 'instance'
-
-    def __init__(self, parent_module, string_name):
-        self.parent_context = parent_module
-        self.string_name = string_name
-
-    def infer(self):
-        return compiled.create(self.parent_context.evaluator, str).execute(
-            param.ValuesArguments([])
-        )
-
-
-class ModuleContext(use_metaclass(CachedMetaClass, context.TreeContext)):
-    api_type = 'module'
-    parent_context = None
-
-    def __init__(self, evaluator, module_node):
-        super(ModuleContext, self).__init__(evaluator, parent_context=None)
-        self.tree_node = module_node
-
-    def get_filters(self, search_global, until_position=None, origin_scope=None):
-        yield ParserTreeFilter(
-            self.evaluator,
-            context=self,
-            until_position=until_position,
-            origin_scope=origin_scope
-        )
-        yield GlobalNameFilter(self, self.tree_node)
-        yield DictFilter(self._sub_modules_dict())
-        yield DictFilter(self._module_attributes_dict())
-        for star_module in self.star_imports():
-            yield next(star_module.get_filters(search_global))
-
-    # I'm not sure if the star import cache is really that effective anymore
-    # with all the other really fast import caches. Recheck. Also we would need
-    # to push the star imports into Evaluator.modules, if we reenable this.
-    @memoize_default([])
-    def star_imports(self):
-        modules = []
-        for i in self.tree_node.imports:
-            if i.is_star_import():
-                name = i.star_import_name()
-                new = imports.infer_import(self, name)
-                for module in new:
-                    if isinstance(module, ModuleContext):
-                        modules += module.star_imports()
-                modules += new
-        return modules
-
-    @memoize_default()
-    def _module_attributes_dict(self):
-        names = ['__file__', '__package__', '__doc__', '__name__']
-        # All the additional module attributes are strings.
-        return dict((n, ModuleAttributeName(self, n)) for n in names)
-
-    @property
-    @memoize_default()
-    def name(self):
-        return ContextName(self, self.tree_node.name)
-
-    def _get_init_directory(self):
-        """
-        :return: The path to the directory of a package. None in case it's not
-                 a package.
-        """
-        for suffix, _, _ in imp.get_suffixes():
-            ending = '__init__' + suffix
-            py__file__ = self.py__file__()
-            if py__file__ is not None and py__file__.endswith(ending):
-                # Remove the ending, including the separator.
-                return self.py__file__()[:-len(ending) - 1]
-        return None
-
-    def py__name__(self):
-        for name, module in self.evaluator.modules.items():
-            if module == self and name != '':
-                return name
-
-        return '__main__'
-
-    def py__file__(self):
-        """
-        In contrast to Python's __file__ can be None.
-        """
-        if self.tree_node.path is None:
-            return None
-
-        return os.path.abspath(self.tree_node.path)
-
-    def py__package__(self):
-        if self._get_init_directory() is None:
-            return re.sub(r'\.?[^\.]+$', '', self.py__name__())
-        else:
-            return self.py__name__()
-
-    def _py__path__(self):
-        search_path = self.evaluator.sys_path
-        init_path = self.py__file__()
-        if os.path.basename(init_path) == '__init__.py':
-            with open(init_path, 'rb') as f:
-                content = common.source_to_unicode(f.read())
-                # these are strings that need to be used for namespace packages,
-                # the first one is ``pkgutil``, the second ``pkg_resources``.
-                options = ('declare_namespace(__name__)', 'extend_path(__path__')
-                if options[0] in content or options[1] in content:
-                    # It is a namespace, now try to find the rest of the
-                    # modules on sys_path or whatever the search_path is.
-                    paths = set()
-                    for s in search_path:
-                        other = os.path.join(s, self.name.string_name)
-                        if os.path.isdir(other):
-                            paths.add(other)
-                    if paths:
-                        return list(paths)
-                    # TODO I'm not sure if this is how nested namespace
-                    # packages work. The tests are not really good enough to
-                    # show that.
-        # Default to this.
-        return [self._get_init_directory()]
-
-    @property
-    def py__path__(self):
-        """
-        Not seen here, since it's a property. The callback actually uses a
-        variable, so use it like::
-
-            foo.py__path__(sys_path)
-
-        In case of a package, this returns Python's __path__ attribute, which
-        is a list of paths (strings).
-        Raises an AttributeError if the module is not a package.
-        """
-        path = self._get_init_directory()
-
-        if path is None:
-            raise AttributeError('Only packages have __path__ attributes.')
-        else:
-            return self._py__path__
-
-    @memoize_default()
-    def _sub_modules_dict(self):
-        """
-        Lists modules in the directory of this module (if this module is a
-        package).
-        """
-        path = self.tree_node.path
-        names = {}
-        if path is not None and path.endswith(os.path.sep + '__init__.py'):
-            mods = pkgutil.iter_modules([os.path.dirname(path)])
-            for module_loader, name, is_pkg in mods:
-                # It's obviously a relative import to the current module.
-                names[name] = imports.SubModuleName(self, name)
-
-        # TODO add something like this in the future, its cleaner than the
-        #   import hacks.
-        # ``os.path`` is a hardcoded exception, because it's a
-        # ``sys.modules`` modification.
-        # if str(self.name) == 'os':
-        #     names.append(Name('path', parent_context=self))
-
-        return names
-
-    def py__class__(self):
-        return compiled.get_special_object(self.evaluator, 'MODULE_CLASS')

jedi/evaluate/signature.py

@@ -0,0 +1,116 @@
+from jedi._compatibility import Parameter
+from jedi.cache import memoize_method
+
+
+class _SignatureMixin(object):
+    def to_string(self):
+        def param_strings():
+            is_positional = False
+            is_kw_only = False
+            for n in self.get_param_names(resolve_stars=True):
+                kind = n.get_kind()
+                is_positional |= kind == Parameter.POSITIONAL_ONLY
+                if is_positional and kind != Parameter.POSITIONAL_ONLY:
+                    yield '/'
+                    is_positional = False
+
+                if kind == Parameter.VAR_POSITIONAL:
+                    is_kw_only = True
+                elif kind == Parameter.KEYWORD_ONLY and not is_kw_only:
+                    yield '*'
+                    is_kw_only = True
+
+                yield n.to_string()
+
+            if is_positional:
+                yield '/'
+
+        s = self.name.string_name + '(' + ', '.join(param_strings()) + ')'
+        annotation = self.annotation_string
+        if annotation:
+            s += ' -> ' + annotation
+        return s
+
+
+class AbstractSignature(_SignatureMixin):
+    def __init__(self, context, is_bound=False):
+        self.context = context
+        self.is_bound = is_bound
+
+    @property
+    def name(self):
+        return self.context.name
+
+    @property
+    def annotation_string(self):
+        return ''
+
+    def get_param_names(self, resolve_stars=False):
+        param_names = self._function_context.get_param_names()
+        if self.is_bound:
+            return param_names[1:]
+        return param_names
+
+    def bind(self, context):
+        raise NotImplementedError
+
+    def __repr__(self):
+        return '<%s: %s, %s>' % (self.__class__.__name__, self.context, self._function_context)
+
+
+class TreeSignature(AbstractSignature):
+    def __init__(self, context, function_context=None, is_bound=False):
+        super(TreeSignature, self).__init__(context, is_bound)
+        self._function_context = function_context or context
+
+    def bind(self, context):
+        return TreeSignature(context, self._function_context, is_bound=True)
+
+    @property
+    def _annotation(self):
+        # Classes don't need annotations, even if __init__ has one. They always
+        # return themselves.
+        if self.context.is_class():
+            return None
+        return self._function_context.tree_node.annotation
+
+    @property
+    def annotation_string(self):
+        a = self._annotation
+        if a is None:
+            return ''
+        return a.get_code(include_prefix=False)
+
+    @memoize_method
+    def get_param_names(self, resolve_stars=False):
+        params = super(TreeSignature, self).get_param_names(resolve_stars=False)
+        if resolve_stars:
+            from jedi.evaluate.star_args import process_params
+            params = process_params(params)
+        return params
+
+
+class BuiltinSignature(AbstractSignature):
+    def __init__(self, context, return_string, is_bound=False):
+        super(BuiltinSignature, self).__init__(context, is_bound)
+        self._return_string = return_string
+
+    @property
+    def annotation_string(self):
+        return self._return_string
+
+    @property
+    def _function_context(self):
+        return self.context
+
+    def bind(self, context):
+        assert not self.is_bound
+        return BuiltinSignature(context, self._return_string, is_bound=True)
+
+
+class SignatureWrapper(_SignatureMixin):
+    def __init__(self, wrapped_signature):
+        self._wrapped_signature = wrapped_signature
+
+    def __getattr__(self, name):
+        return getattr(self._wrapped_signature, name)