v13.3 release notes

see comment in 8d313e014f

.coveragerc

@@ -1,19 +1,15 @@
 [run]
 omit =
     jedi/_compatibility.py
-    jedi/evaluate/site.py
+    jedi/evaluate/compiled/subprocess/__main__.py
+    jedi/__main__.py
+    # Is statically analyzed and not actually executed.
+    jedi/evaluate/jedi_typing.py
+    # For now this is not being used.
+    jedi/refactoring.py
 
 [report]
 # Regexes for lines to exclude from consideration
 exclude_lines =
     # Don't complain about missing debug-only code:
     def __repr__
-    if self\.debug
-
-    # Don't complain if tests don't hit defensive assertion code:
-    raise AssertionError
-    raise NotImplementedError
-
-    # Don't complain if non-runnable code isn't run:
-    if 0:
-    if __name__ == .__main__.:

.gitignore

@@ -5,9 +5,11 @@
 .tox
 .coveralls.yml
 .coverage
+.idea
 /build/
 /docs/_build/
 /dist/
 jedi.egg-info/
 record.json
 /.cache/
+/.pytest_cache

.travis.yml

@@ -1,30 +1,57 @@
 language: python
-sudo: false
+sudo: true
 python:
-  - 2.6
   - 2.7
-  - 3.3
   - 3.4
   - 3.5
   - 3.6
-  - pypy
+
+env:
+  - JEDI_TEST_ENVIRONMENT=27
+  - JEDI_TEST_ENVIRONMENT=34
+  - JEDI_TEST_ENVIRONMENT=35
+  - JEDI_TEST_ENVIRONMENT=36
+
+addons:
+  apt:
+    packages:
+     # Required to properly create a virtual environment with system Python 3.4.
+     - python3.4-venv
+
 matrix:
   allow_failures:
     - python: pypy
-    - env: TOXENV=cov
     - env: TOXENV=sith
+    - python: 3.7-dev
   include:
-    - python: 3.5
-      env: TOXENV=cov
-    - python: 3.5
+    - python: 3.6
+      env:
+        - TOXENV=cov
+        - JEDI_TEST_ENVIRONMENT=36
+    - python: 3.6
       env: TOXENV=sith
+    # For now ignore pypy, there are so many issues that we don't really need
+    # to run it.
+    #- python: pypy
+    - python: "3.7-dev"
+before_install:
+    - ./travis_install.sh
+    # Need to add the path to the Python versions in the end. This might add
+    # something twice, but it doesn't really matter, because they are appended.
+    - export PATH=$PATH:/opt/python/3.5/bin
+    # 3.6 was not installed manually, but already is on the system. However
+    # it's not on path (unless 3.6 is selected).
+    - export PATH=$PATH:/opt/python/3.6/bin
 install:
     - pip install --quiet tox-travis
 script:
     - tox
 after_script:
-    - if [ $TOXENV == "cov" ]; then
-      pip install --quiet coveralls;
-      coveralls;
+    - |
+      if [ $TOXENV == "cov" ]; then
+        pip install --quiet codecov coveralls
+        coverage xml
+        coverage report -m
+        coveralls
+        bash <(curl -s https://codecov.io/bash) -X gcov -X coveragepy -X search -X fix -X xcode -f coverage.xml
       fi
-

AUTHORS.txt

@@ -1,4 +1,4 @@
-Main Authors
+Main Authors
 ============
 
 David Halter (@davidhalter) <davidhalter88@gmail.com>
@@ -44,6 +44,12 @@ Mathias Rav (@Mortal) <rav@cs.au.dk>
 Daniel Fiterman (@dfit99) <fitermandaniel2@gmail.com>
 Simon Ruggier (@sruggier)
 Élie Gouzien (@ElieGouzien)
-
+Robin Roth (@robinro)
+Malte Plath (@langsamer)
+Anton Zub (@zabulazza)
+Maksim Novikov (@m-novikov) <mnovikov.work@gmail.com>
+Tobias Rzepka (@TobiasRzepka)
+micbou (@micbou)
+Dima Gerasimov (@karlicoss) <karlicoss@gmail.com>
 
 Note: (@user) means a github user name.

CHANGELOG.rst

@@ -3,6 +3,59 @@
 Changelog
 ---------
 
+0.13.3 (2019-02-24)
++++++++++++++++++++
+
+- Fixed an issue with embedded Pytho, see https://github.com/davidhalter/jedi-vim/issues/870
+
+0.13.2 (2018-12-15)
++++++++++++++++++++
+
+- Fixed a bug that led to Jedi spawning a lot of subprocesses.
+
+0.13.1 (2018-10-02)
++++++++++++++++++++
+
+- Bugfixes, because tensorflow completions were still slow.
+
+0.13.0 (2018-10-02)
++++++++++++++++++++
+
+- A small release. Some bug fixes.
+- Remove Python 3.3 support. Python 3.3 support has been dropped by the Python
+  foundation.
+- Default environments are now using the same Python version as the Python
+  process. In 0.12.x, we used to load the latest Python version on the system.
+- Added ``include_builtins`` as a parameter to usages.
+- ``goto_assignments`` has a new ``follow_builtin_imports`` parameter that
+  changes the previous behavior slightly.
+
+0.12.1 (2018-06-30)
++++++++++++++++++++
+
+- This release forces you to upgrade parso. If you don't, nothing will work
+  anymore. Otherwise changes should be limited to bug fixes. Unfortunately Jedi
+  still uses a few internals of parso that make it hard to keep compatibility
+  over multiple releases. Parso >=0.3.0 is going to be needed.
+
+0.12.0 (2018-04-15)
++++++++++++++++++++
+
+- Virtualenv/Environment support
+- F-String Completion/Goto Support
+- Cannot crash with segfaults anymore
+- Cleaned up import logic
+- Understand async/await and autocomplete it (including async generators)
+- Better namespace completions
+- Passing tests for Windows (including CI for Windows)
+- Remove Python 2.6 support
+
+0.11.1 (2017-12-14)
++++++++++++++++++++
+
+- Parso update - the caching layer was broken
+- Better usages - a lot of internal code was ripped out and improved.
+
 0.11.0 (2017-09-20)
 +++++++++++++++++++
 

CONTRIBUTING.md

@@ -5,4 +5,4 @@ Pull Requests are great.
  3. Add your name to AUTHORS.txt
  4. Push to your fork and submit a pull request.
 
-**Try to use the PEP8 style guide.**
+**Try to use the PEP8 style guide** (and it's ok to have a line length of 100 characters).

README.rst

@@ -2,16 +2,28 @@
 Jedi - an awesome autocompletion/static analysis library for Python
 ###################################################################
 
-.. image:: https://secure.travis-ci.org/davidhalter/jedi.png?branch=master
-    :target: http://travis-ci.org/davidhalter/jedi
-    :alt: Travis-CI build status
+.. image:: https://img.shields.io/pypi/v/jedi.svg?style=flat
+    :target: https://pypi.python.org/pypi/jedi
+    :alt: PyPI version
 
-.. image:: https://coveralls.io/repos/davidhalter/jedi/badge.png?branch=master
+.. image:: https://img.shields.io/pypi/pyversions/jedi.svg
+    :target: https://pypi.python.org/pypi/jedi
+    :alt: Supported Python versions
+
+.. image:: https://travis-ci.org/davidhalter/jedi.svg?branch=master
+    :target: https://travis-ci.org/davidhalter/jedi
+    :alt: Linux Tests
+
+.. image:: https://ci.appveyor.com/api/projects/status/mgva3bbawyma1new/branch/master?svg=true
+    :target: https://ci.appveyor.com/project/davidhalter/jedi/branch/master
+    :alt: Windows Tests
+
+.. image:: https://coveralls.io/repos/davidhalter/jedi/badge.svg?branch=master
     :target: https://coveralls.io/r/davidhalter/jedi
-    :alt: Coverage Status
+    :alt: Coverage status
 
 
-*If you have specific questions, please add an issue or ask on* `stackoverflow
+*If you have specific questions, please add an issue or ask on* `Stack Overflow
 <https://stackoverflow.com/questions/tagged/python-jedi>`_ *with the label* ``python-jedi``.
 
 
@@ -25,7 +37,7 @@ related names and to list all names in a Python file and infer them. Jedi
 understands docstrings and you can use Jedi autocompletion in your REPL as
 well.
 
-Jedi uses a very simple API to connect with IDE's. There's a reference
+Jedi uses a very simple API to connect with IDEs. There's a reference
 implementation as a `VIM-Plugin <https://github.com/davidhalter/jedi-vim>`_,
 which uses Jedi's autocompletion.  We encourage you to use Jedi in your IDEs.
 It's really easy.
@@ -39,13 +51,12 @@ Jedi can currently be used with the following editors/projects:
 - Kate_ version 4.13+ supports it natively, you have to enable it, though. [`proof
   <https://projects.kde.org/projects/kde/applications/kate/repository/show?rev=KDE%2F4.13>`_]
 - Atom_ (autocomplete-python-jedi_)
-- SourceLair_
 - `GNOME Builder`_ (with support for GObject Introspection)
-- `Visual Studio Code`_ (via `Python Extension <https://marketplace.visualstudio.com/items?itemName=donjayamanne.python>`_)
+- `Visual Studio Code`_ (via `Python Extension <https://marketplace.visualstudio.com/items?itemName=ms-python.python>`_)
 - Gedit (gedi_)
 - wdb_ - Web Debugger
 - `Eric IDE`_ (Available as a plugin)
-- `Ipython 6.0.0+ <http://ipython.readthedocs.io/en/stable/whatsnew/version6.html>`_
+- `IPython 6.0.0+ <https://ipython.readthedocs.io/en/stable/whatsnew/version6.html>`_
 
 and many more!
 
@@ -72,7 +83,11 @@ Get the latest version from `github <https://github.com/davidhalter/jedi>`_
 Docs are available at `https://jedi.readthedocs.org/en/latest/
 <https://jedi.readthedocs.org/en/latest/>`_. Pull requests with documentation
 enhancements and/or fixes are awesome and most welcome. Jedi uses `semantic
-versioning <http://semver.org/>`_.
+versioning <https://semver.org/>`_.
+
+If you want to stay up-to-date (News / RFCs), please subscribe to this `github
+thread <https://github.com/davidhalter/jedi/issues/1063>`_.:
+
 
 
 Installation
@@ -96,8 +111,10 @@ understands, see: `Features
 <https://jedi.readthedocs.org/en/latest/docs/features.html>`_. A list of
 caveats can be found on the same page.
 
-You can run Jedi on cPython 2.6, 2.7, 3.3, 3.4 or 3.5 but it should also
-understand/parse code older than those versions.
+You can run Jedi on CPython 2.7 or 3.4+ but it should also
+understand/parse code older than those versions. Additionally you should be able
+to use `Virtualenvs <https://jedi.readthedocs.org/en/latest/docs/api.html#environments>`_
+very well.
 
 Tips on how to use Jedi efficiently can be found `here
 <https://jedi.readthedocs.org/en/latest/docs/features.html#recipes>`_.
@@ -105,7 +122,7 @@ Tips on how to use Jedi efficiently can be found `here
 API
 ---
 
-You can find the documentation for the `API here <https://jedi.readthedocs.org/en/latest/docs/plugin-api.html>`_.
+You can find the documentation for the `API here <https://jedi.readthedocs.org/en/latest/docs/api.html>`_.
 
 
 Autocompletion / Goto / Pydoc
@@ -123,7 +140,7 @@ The returned objects are very powerful and really all you might need.
 Autocompletion in your REPL (IPython, etc.)
 -------------------------------------------
 
-Starting with Ipython `6.0.0` Jedi is a dependency of IPython. Autocompletion
+Starting with IPython `6.0.0` Jedi is a dependency of IPython. Autocompletion
 in IPython is therefore possible without additional configuration.
 
 It's possible to have Jedi autocompletion in REPL modes - `example video <https://vimeo.com/122332037>`_.
@@ -178,7 +195,7 @@ Tests are also run automatically on `Travis CI
 <https://travis-ci.org/davidhalter/jedi/>`_.
 
 For more detailed information visit the `testing documentation
-<https://jedi.readthedocs.org/en/latest/docs/testing.html>`_
+<https://jedi.readthedocs.org/en/latest/docs/testing.html>`_.
 
 
 Acknowledgements
@@ -193,7 +210,7 @@ Acknowledgements
 
 
 .. _jedi-vim: https://github.com/davidhalter/jedi-vim
-.. _youcompleteme: http://valloric.github.io/YouCompleteMe/
+.. _youcompleteme: https://valloric.github.io/YouCompleteMe/
 .. _deoplete-jedi: https://github.com/zchee/deoplete-jedi
 .. _completor.vim: https://github.com/maralla/completor.vim
 .. _Jedi.el: https://github.com/tkf/emacs-jedi
@@ -205,11 +222,10 @@ Acknowledgements
 .. _anaconda: https://github.com/DamnWidget/anaconda
 .. _wdb: https://github.com/Kozea/wdb
 .. _TextMate: https://github.com/lawrenceakka/python-jedi.tmbundle
-.. _Kate: http://kate-editor.org
+.. _Kate: https://kate-editor.org
 .. _Atom: https://atom.io/
 .. _autocomplete-python-jedi: https://atom.io/packages/autocomplete-python-jedi
-.. _SourceLair: https://www.sourcelair.com
 .. _GNOME Builder: https://wiki.gnome.org/Apps/Builder
 .. _Visual Studio Code: https://code.visualstudio.com/
 .. _gedi: https://github.com/isamert/gedi
-.. _Eric IDE: http://eric-ide.python-projects.org
+.. _Eric IDE: https://eric-ide.python-projects.org

appveyor.yml

@@ -0,0 +1,58 @@
+environment:
+  matrix:
+  - TOXENV: py27
+    PYTHON_PATH: C:\Python27
+    JEDI_TEST_ENVIRONMENT: 27
+  - TOXENV: py27
+    PYTHON_PATH: C:\Python27
+    JEDI_TEST_ENVIRONMENT: 34
+  - TOXENV: py27
+    PYTHON_PATH: C:\Python27
+    JEDI_TEST_ENVIRONMENT: 35
+  - TOXENV: py27
+    PYTHON_PATH: C:\Python27
+    JEDI_TEST_ENVIRONMENT: 36
+
+  - TOXENV: py34
+    PYTHON_PATH: C:\Python34
+    JEDI_TEST_ENVIRONMENT: 27
+  - TOXENV: py34
+    PYTHON_PATH: C:\Python34
+    JEDI_TEST_ENVIRONMENT: 34
+  - TOXENV: py34
+    PYTHON_PATH: C:\Python34
+    JEDI_TEST_ENVIRONMENT: 35
+  - TOXENV: py34
+    PYTHON_PATH: C:\Python34
+    JEDI_TEST_ENVIRONMENT: 36
+
+  - TOXENV: py35
+    PYTHON_PATH: C:\Python35
+    JEDI_TEST_ENVIRONMENT: 27
+  - TOXENV: py35
+    PYTHON_PATH: C:\Python35
+    JEDI_TEST_ENVIRONMENT: 34
+  - TOXENV: py35
+    PYTHON_PATH: C:\Python35
+    JEDI_TEST_ENVIRONMENT: 35
+  - TOXENV: py35
+    PYTHON_PATH: C:\Python35
+    JEDI_TEST_ENVIRONMENT: 36
+
+  - TOXENV: py36
+    PYTHON_PATH: C:\Python36
+    JEDI_TEST_ENVIRONMENT: 27
+  - TOXENV: py36
+    PYTHON_PATH: C:\Python36
+    JEDI_TEST_ENVIRONMENT: 34
+  - TOXENV: py36
+    PYTHON_PATH: C:\Python36
+    JEDI_TEST_ENVIRONMENT: 35
+  - TOXENV: py36
+    PYTHON_PATH: C:\Python36
+    JEDI_TEST_ENVIRONMENT: 36
+install:
+  - set PATH=%PYTHON_PATH%;%PYTHON_PATH%\Scripts;%PATH%
+  - pip install tox
+build_script:
+  - tox

conftest.py

@@ -1,11 +1,21 @@
 import tempfile
 import shutil
+import os
+from functools import partial
 
 import pytest
 
 import jedi
+from jedi.api.environment import get_system_environment, InterpreterEnvironment
+from jedi._compatibility import py_version
 
-collect_ignore = ["setup.py"]
+collect_ignore = [
+    'setup.py',
+    '__main__.py',
+    'jedi/evaluate/compiled/subprocess/__main__.py',
+    'build/',
+    'test/examples',
+]
 
 
 # The following hooks (pytest_configure, pytest_unconfigure) are used
@@ -29,6 +39,12 @@ def pytest_addoption(parser):
     parser.addoption("--warning-is-error", action='store_true',
                      help="Warnings are treated as errors.")
 
+    parser.addoption("--env", action='store',
+                     help="Execute the tests in that environment (e.g. 35 for python3.5).")
+    parser.addoption("--interpreter-env", "-I", action='store_true',
+                     help="Don't use subprocesses to guarantee having safe "
+                          "code execution. Useful for debugging.")
+
 
 def pytest_configure(config):
     global jedi_cache_directory_orig, jedi_cache_directory_temp
@@ -70,3 +86,36 @@ def clean_jedi_cache(request):
     def restore():
         settings.cache_directory = old
         shutil.rmtree(tmp)
+
+
+@pytest.fixture(scope='session')
+def environment(request):
+    if request.config.option.interpreter_env:
+        return InterpreterEnvironment()
+
+    version = request.config.option.env
+    if version is None:
+        version = os.environ.get('JEDI_TEST_ENVIRONMENT', str(py_version))
+
+    return get_system_environment(version[0] + '.' + version[1:])
+
+
+@pytest.fixture(scope='session')
+def Script(environment):
+    return partial(jedi.Script, environment=environment)
+
+
+@pytest.fixture(scope='session')
+def has_typing(environment):
+    if environment.version_info >= (3, 5, 0):
+        # This if is just needed to avoid that tests ever skip way more than
+        # they should for all Python versions.
+        return True
+
+    script = jedi.Script('import typing', environment=environment)
+    return bool(script.goto_definitions())
+
+
+@pytest.fixture(scope='session')
+def jedi_path():
+    return os.path.dirname(__file__)

deploy-master.sh

@@ -36,7 +36,7 @@ if [[ $tag_ref ]]; then
         exit 1
     fi
 else
-    git tag $tag
+    git tag -a $tag
     git push --tags
 fi
 
@@ -48,5 +48,5 @@ python setup.py sdist bdist_wheel
 twine upload dist/*
 
 cd $BASE_DIR
-# Back in the development directory fetch tags.
-git fetch --tags
+# The tags have been pushed to this repo. Push the tags to github, now.
+git push --tags

docs/_templates/ghbuttons.html

@@ -1,4 +1,4 @@
 <h3>Github</h3>
-<iframe src="http://ghbtns.com/github-btn.html?user=davidhalter&repo=jedi&type=watch&count=true&size=large"
+<iframe src="https://ghbtns.com/github-btn.html?user=davidhalter&repo=jedi&type=watch&count=true&size=large"
         frameborder="0" scrolling="0" width="170" height="30" allowtransparency="true"></iframe>
 <br><br>

docs/_themes/flask/layout.html

@@ -19,7 +19,6 @@
 {% endblock %}
 {%- block footer %}
   <div class="footer">
-    &copy; Copyright {{ copyright }}.
     Created using <a href="http://sphinx.pocoo.org/">Sphinx</a>.
   </div>
   {% if pagename == 'index' %}

docs/conf.py

@@ -45,7 +45,7 @@ master_doc = 'index'
 
 # General information about the project.
 project = u'Jedi'
-copyright = u'2012 - {today.year}, Jedi contributors'.format(today=datetime.date.today())
+copyright = u'jedi contributors'
 
 import jedi
 from jedi.utils import version_info
@@ -274,7 +274,7 @@ autodoc_default_flags = []
 # -- Options for intersphinx module --------------------------------------------
 
 intersphinx_mapping = {
-    'http://docs.python.org/': None,
+    'https://docs.python.org/': None,
 }
 
 

docs/docs/api-classes.rst

@@ -1,6 +1,6 @@
 .. include:: ../global.rst
 
-.. _plugin-api-classes:
+.. _api-classes:
 
 API Return Classes
 ------------------

docs/docs/api.rst

@@ -1,7 +1,7 @@
 .. include:: ../global.rst
 
-The Plugin API
-==============
+API Overview
+============
 
 .. currentmodule:: jedi
 
@@ -11,7 +11,7 @@ editors/IDE autocompletion
 If you want to use |jedi|, you first need to ``import jedi``.  You then have
 direct access to the :class:`.Script`. You can then call the functions
 documented here. These functions return :ref:`API classes
-<plugin-api-classes>`.
+<api-classes>`.
 
 
 Deprecations
@@ -24,16 +24,48 @@ The deprecation process is as follows:
    the deprecated functionality.
 
 
-API documentation
+API Documentation
 -----------------
 
-API Interface
-~~~~~~~~~~~~~
+The API consists of a few different parts:
 
-.. automodule:: jedi.api
+- The main starting points for completions/goto: :class:`.Script` and :class:`.Interpreter`
+- Helpful functions: :func:`.names`, :func:`.preload_module` and
+  :func:`.set_debug_function`
+- :ref:`API Result Classes <api-classes>`
+- :ref:`Python Versions/Virtualenv Support <environments>` with functions like
+  :func:`.find_system_environments` and :func:`.find_virtualenvs`
+
+.. _api:
+
+Static Analysis Interface
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. automodule:: jedi
+
+.. autoclass:: jedi.Script
     :members:
-    :undoc-members:
+.. autoclass:: jedi.Interpreter
+    :members:
+.. autofunction:: jedi.names
+.. autofunction:: jedi.preload_module
+.. autofunction:: jedi.set_debug_function
 
+.. _environments:
+
+Environments
+~~~~~~~~~~~~
+
+.. automodule:: jedi.api.environment
+
+.. autofunction:: jedi.find_system_environments
+.. autofunction:: jedi.find_virtualenvs
+.. autofunction:: jedi.get_system_environment
+.. autofunction:: jedi.create_environment
+.. autofunction:: jedi.get_default_environment
+.. autoexception:: jedi.InvalidPythonEnvironment
+.. autoclass:: jedi.api.environment.Environment
+    :members:
 
 Examples
 --------

docs/docs/development.rst

@@ -7,7 +7,7 @@ Jedi Development
 
 .. note:: This documentation is for Jedi developers who want to improve Jedi
     itself, but have no idea how Jedi works. If you want to use Jedi for 
-    your IDE, look at the `plugin api <plugin-api.html>`_.
+    your IDE, look at the `plugin api <api.html>`_.
 
 
 Introduction
@@ -54,31 +54,15 @@ because that's where all the magic happens. I need to introduce the :ref:`parser
 
 .. _parser:
 
-Parser (parser/__init__.py)
+Parser
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-.. automodule:: jedi.parser
+Jedi used to have it's internal parser, however this is now a separate project
+and is called `parso <http://parso.readthedocs.io>`_.
 
-Parser Tree (parser/tree.py)
-++++++++++++++++++++++++++++++++++++++++++++++++
-
-.. automodule:: jedi.parser.tree
-
-Class inheritance diagram:
-
-.. inheritance-diagram::
-   Module
-   Class
-   Function
-   Lambda
-   Flow
-   ForStmt
-   Import
-   ExprStmt
-   Param
-   Name
-   CompFor
-   :parts: 1
+The parser creates a syntax tree that |jedi| analyses and tries to understand.
+The grammar that this parsers uses is very similar to the official Python
+`grammar files <https://docs.python.org/3/reference/grammar.html>`_.
 
 .. _evaluate:
 
@@ -87,16 +71,16 @@ Evaluation of python code (evaluate/__init__.py)
 
 .. automodule:: jedi.evaluate
 
-Evaluation Representation (evaluate/representation.py)
+Evaluation Contexts (evaluate/base_context.py)
 ++++++++++++++++++++++++++++++++++++++++++++++++++++++
 
-.. automodule:: jedi.evaluate.representation
+.. automodule:: jedi.evaluate.base_context
 
 .. inheritance-diagram::
-   jedi.evaluate.instance.TreeInstance
-   jedi.evaluate.representation.ClassContext
-   jedi.evaluate.representation.FunctionContext
-   jedi.evaluate.representation.FunctionExecutionContext
+   jedi.evaluate.context.instance.TreeInstance
+   jedi.evaluate.context.klass.ClassContext
+   jedi.evaluate.context.function.FunctionContext
+   jedi.evaluate.context.function.FunctionExecutionContext
    :parts: 1
 
 
@@ -110,11 +94,11 @@ Name resolution (evaluate/finder.py)
 
 .. _dev-api:
 
-API (api.py and api_classes.py)
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+API (api/__init__.py and api/classes.py)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 The API has been designed to be as easy to use as possible. The API
-documentation can be found `here <plugin-api.html>`_. The API itself contains
+documentation can be found `here <api.html>`_. The API itself contains
 little code that needs to be mentioned here. Generally I'm trying to be
 conservative with the API.  I'd rather not add new API features if they are not
 necessary, because it's much harder to deprecate stuff than to add it later.
@@ -129,7 +113,6 @@ Core Extensions is a summary of the following topics:
 
 - :ref:`Iterables & Dynamic Arrays <iterables>`
 - :ref:`Dynamic Parameters <dynamic>`
-- :ref:`Diff Parser <diff-parser>`
 - :ref:`Docstrings <docstrings>`
 - :ref:`Refactoring <refactoring>`
 
@@ -139,13 +122,13 @@ without some features.
 
 .. _iterables:
 
-Iterables & Dynamic Arrays (evaluate/iterable.py)
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Iterables & Dynamic Arrays (evaluate/context/iterable.py)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 To understand Python on a deeper level, |jedi| needs to understand some of the
 dynamic features of Python like lists that are filled after creation:
 
-.. automodule:: jedi.evaluate.iterable
+.. automodule:: jedi.evaluate.context.iterable
 
 
 .. _dynamic:
@@ -156,13 +139,6 @@ Parameter completion (evaluate/dynamic.py)
 .. automodule:: jedi.evaluate.dynamic
 
 
-.. _diff-parser:
-
-Diff Parser (parser/diff.py)
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. automodule:: jedi.parser.python.diff
-
 .. _docstrings:
 
 Docstrings (evaluate/docstrings.py)

docs/docs/features.rst

@@ -20,11 +20,11 @@ make it work.
 General Features
 ----------------
 
-- python 2.6+ and 3.3+ support
-- ignores syntax errors and wrong indentation
-- can deal with complex module / function / class structures
-- virtualenv support
-- can infer function arguments from sphinx, epydoc and basic numpydoc docstrings,
+- Python 2.7 and 3.4+ support
+- Ignores syntax errors and wrong indentation
+- Can deal with complex module / function / class structures
+- Great Virtualenv support
+- Can infer function arguments from sphinx, epydoc and basic numpydoc docstrings,
   and PEP0484-style type hints (:ref:`type hinting <type-hinting>`)
 
 
@@ -47,24 +47,22 @@ Supported Python Features
 - (nested) list comprehensions / ternary expressions
 - relative imports
 - ``getattr()`` / ``__getattr__`` / ``__getattribute__``
-- function annotations (py3k feature, are ignored right now, but being parsed.
-  I don't know what to do with them.)
+- function annotations
 - class decorators (py3k feature, are being ignored too, until I find a use
   case, that doesn't work with |jedi|)
 - simple/usual ``sys.path`` modifications
 - ``isinstance`` checks for if/while/assert
-- namespace packages (includes ``pkgutil`` and ``pkg_resources`` namespaces)
+- namespace packages (includes ``pkgutil``, ``pkg_resources`` and PEP420 namespaces)
 - Django / Flask / Buildout support
 
 
-Unsupported Features
---------------------
+Not Supported
+-------------
 
 Not yet implemented:
 
 - manipulations of instances outside the instance variables without using
   methods
-- implicit namespace packages (Python 3.3+, `PEP 420 <https://www.python.org/dev/peps/pep-0420/>`_)
 
 Will probably never be implemented:
 
@@ -77,21 +75,6 @@ Will probably never be implemented:
 Caveats
 -------
 
-**Malformed Syntax**
-
-Syntax errors and other strange stuff may lead to undefined behaviour of the
-completion. |jedi| is **NOT** a Python compiler, that tries to correct you. It
-is a tool that wants to help you. But **YOU** have to know Python, not |jedi|.
-
-**Legacy Python 2 Features**
-
-This framework should work for both Python 2/3. However, some things were just
-not as *pythonic* in Python 2 as things should be. To keep things simple, some
-older Python 2 features have been left out:
-
-- Classes: Always Python 3 like, therefore all classes inherit from ``object``.
-- Generators: No ``next()`` method. The ``__next__()`` method is used instead.
-
 **Slow Performance**
 
 Importing ``numpy`` can be quite slow sometimes, as well as loading the
@@ -103,7 +86,7 @@ etc.
 **Security**
 
 Security is an important issue for |jedi|. Therefore no Python code is
-executed.  As long as you write pure python, everything is evaluated
+executed.  As long as you write pure Python, everything is evaluated
 statically. But: If you use builtin modules (``c_builtin``) there is no other
 option than to execute those modules. However: Execute isn't that critical (as
 e.g. in pythoncomplete, which used to execute *every* import!), because it
@@ -132,8 +115,7 @@ one of the following docstring/annotation syntax styles:
 
 https://www.python.org/dev/peps/pep-0484/
 
-function annotations (python 3 only; python 2 function annotations with
-comments in planned but not yet implemented)
+function annotations
 
 ::
 
@@ -144,7 +126,7 @@ comments in planned but not yet implemented)
         node.| # complete here
 
 
-assignment, for-loop and with-statement type hints (all python versions).
+assignment, for-loop and with-statement type hints (all Python versions).
 Note that the type hints must be on the same line as the statement
 
 ::
@@ -157,7 +139,7 @@ Note that the type hints must be on the same line as the statement
         print(f + 3)
 
 Most of the features in PEP-0484 are supported including the typing module
-(for python < 3.5 you have to do ``pip install typing`` to use these),
+(for Python < 3.5 you have to do ``pip install typing`` to use these),
 and forward references.
 
 Things that are missing (and this is not an exhaustive list; some of these
@@ -172,7 +154,7 @@ are planned, others might be hard to implement and provide little worth):
 
 **Sphinx style**
 
-http://sphinx-doc.org/domains.html#info-field-lists
+http://www.sphinx-doc.org/en/stable/domains.html#info-field-lists
 
 ::
 
@@ -237,7 +219,7 @@ A little history
 
 The Star Wars Jedi are awesome. My Jedi software tries to imitate a little bit
 of the precognition the Jedi have. There's even an awesome `scene
-<http://www.youtube.com/watch?v=5BDO3pyavOY>`_ of Monty Python Jedis :-).
+<https://youtu.be/yHRJLIf7wMU>`_ of Monty Python Jedis :-).
 
 But actually the name hasn't so much to do with Star Wars. It's part of my
 second name.

docs/docs/installation.rst

@@ -49,7 +49,7 @@ Debian
 ~~~~~~
 
 Debian packages are available in the `unstable repository
-<http://packages.debian.org/search?keywords=python%20jedi>`__.
+<https://packages.debian.org/search?keywords=python%20jedi>`__.
 
 Others
 ~~~~~~

docs/docs/parser.rst

@@ -1,36 +0,0 @@
-.. _xxx:
-
-Parser Tree
-===========
-
-Usage
------
-
-.. automodule:: jedi.parser.python
-    :members:
-    :undoc-members:
-
-
-Parser Tree Base Class
-----------------------
-
-All nodes and leaves have these methods/properties:
-
-.. autoclass:: jedi.parser.tree.NodeOrLeaf
-    :members:
-    :undoc-members:
-
-
-Python Parser Tree
-------------------
-
-.. automodule:: jedi.parser.python.tree
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
-
-Utility
--------
-
-.. autofunction:: jedi.parser.tree.search_ancestor

docs/docs/static_analysis.rst

@@ -1,106 +0,0 @@
-
-This file is the start of the documentation of how static analysis works.
-
-Below is a list of parser names that are used within nodes_to_execute.
-
------------- cared for:
-global_stmt
-exec_stmt  # no priority
-assert_stmt
-if_stmt
-while_stmt
-for_stmt
-try_stmt
-(except_clause)
-with_stmt
-(with_item)
-(with_var)
-print_stmt
-del_stmt
-return_stmt
-raise_stmt
-yield_expr
-file_input
-funcdef
-param
-old_lambdef
-lambdef
-import_name
-import_from
-(import_as_name)
-(dotted_as_name)
-(import_as_names)
-(dotted_as_names)
-(dotted_name)
-classdef
-comp_for
-(comp_if) ?
-decorator
-
------------ add basic
-test
-or_test
-and_test
-not_test
-expr
-xor_expr
-and_expr
-shift_expr
-arith_expr
-term
-factor
-power
-atom
-comparison
-expr_stmt
-testlist
-testlist1
-testlist_safe
-
------------ special care:
-# mostly depends on how we handle the other ones.
-testlist_star_expr  # should probably just work with expr_stmt
-star_expr
-exprlist  # just ignore? then names are just resolved. Strange anyway, bc expr is not really allowed in the list, typically.
-
------------ ignore:
-suite
-subscriptlist
-subscript
-simple_stmt
-?? sliceop     # can probably just be added.
-testlist_comp  # prob ignore and care about it with atom.
-dictorsetmaker
-trailer
-decorators
-decorated
-# always execute function arguments? -> no problem with stars.
-# Also arglist and argument are different in different grammars.
-arglist
-argument
-
-
------------ remove:
-tname      # only exists in current Jedi parser. REMOVE!
-tfpdef     # python 2: tuple assignment; python 3: annotation
-vfpdef     # reduced in python 3 and therefore not existing.
-tfplist    # not in 3
-vfplist    # not in 3
-
---------- not existing with parser reductions.
-small_stmt
-import_stmt
-flow_stmt
-compound_stmt
-stmt
-pass_stmt
-break_stmt
-continue_stmt
-comp_op
-augassign
-old_test
-typedargslist  # afaik becomes [param]
-varargslist    # dito
-vname
-comp_iter
-test_nocond

docs/docs/usage.rst

@@ -56,10 +56,6 @@ Atom:
 
 - autocomplete-python-jedi_
 
-SourceLair:
-
-- SourceLair_
-
 GNOME Builder:
 
 - `GNOME Builder`_ `supports it natively
@@ -103,7 +99,7 @@ Using a custom ``$HOME/.pythonrc.py``
 .. autofunction:: jedi.utils.setup_readline
 
 .. _jedi-vim: https://github.com/davidhalter/jedi-vim
-.. _youcompleteme: http://valloric.github.io/YouCompleteMe/
+.. _youcompleteme: https://valloric.github.io/YouCompleteMe/
 .. _deoplete-jedi: https://github.com/zchee/deoplete-jedi
 .. _Jedi.el: https://github.com/tkf/emacs-jedi
 .. _elpy: https://github.com/jorgenschaefer/elpy
@@ -113,10 +109,9 @@ Using a custom ``$HOME/.pythonrc.py``
 .. _SynJedi: http://uvviewsoft.com/synjedi/
 .. _wdb: https://github.com/Kozea/wdb
 .. _TextMate: https://github.com/lawrenceakka/python-jedi.tmbundle
-.. _kate: http://kate-editor.org/
+.. _kate: https://kate-editor.org/
 .. _autocomplete-python-jedi: https://atom.io/packages/autocomplete-python-jedi
-.. _SourceLair: https://www.sourcelair.com
 .. _GNOME Builder: https://wiki.gnome.org/Apps/Builder/
 .. _gedi: https://github.com/isamert/gedi
-.. _Eric IDE: http://eric-ide.python-projects.org
+.. _Eric IDE: https://eric-ide.python-projects.org
 .. _Python Extension: https://marketplace.visualstudio.com/items?itemName=donjayamanne.python

docs/index.rst

@@ -23,8 +23,8 @@ Docs
    docs/usage
    docs/installation
    docs/features
-   docs/plugin-api
-   docs/plugin-api-classes
+   docs/api
+   docs/api-classes
    docs/settings
    docs/development
    docs/testing
@@ -37,4 +37,4 @@ Resources
 
 - `Source Code on Github <https://github.com/davidhalter/jedi>`_
 - `Travis Testing <https://travis-ci.org/davidhalter/jedi>`_
-- `Python Package Index <http://pypi.python.org/pypi/jedi/>`_
+- `Python Package Index <https://pypi.python.org/pypi/jedi/>`_

jedi/__init__.py

@@ -23,7 +23,7 @@ example for the autocompletion feature:
 ... datetime.da'''
 >>> script = jedi.Script(source, 3, len('datetime.da'), 'example.py')
 >>> script
-<Script: 'example.py'>
+<Script: 'example.py' ...>
 >>> completions = script.completions()
 >>> completions                                         #doctest: +ELLIPSIS
 [<Completion: date>, <Completion: datetime>, ...]
@@ -36,8 +36,12 @@ As you see Jedi is pretty simple and allows you to concentrate on writing a
 good text editor, while still having very good IDE features for Python.
 """
 
-__version__ = '0.11.0'
+__version__ = '0.13.3'
 
 from jedi.api import Script, Interpreter, set_debug_function, \
     preload_module, names
 from jedi import settings
+from jedi.api.environment import find_virtualenvs, find_system_environments, \
+    get_default_environment, InvalidPythonEnvironment, create_environment, \
+    get_system_environment
+from jedi.api.exceptions import InternalError

jedi/_compatibility.py

@@ -1,25 +1,22 @@
 """
-To ensure compatibility from Python ``2.6`` - ``3.3``, a module has been
+To ensure compatibility from Python ``2.7`` - ``3.x``, a module has been
 created. Clearly there is huge need to use conforming syntax.
 """
+import errno
 import sys
-import imp
 import os
 import re
 import pkgutil
 import warnings
+import inspect
+import subprocess
 try:
     import importlib
 except ImportError:
     pass
 
-# Cannot use sys.version.major and minor names, because in Python 2.6 it's not
-# a namedtuple.
 is_py3 = sys.version_info[0] >= 3
-is_py33 = is_py3 and sys.version_info[1] >= 3
-is_py34 = is_py3 and sys.version_info[1] >= 4
 is_py35 = is_py3 and sys.version_info[1] >= 5
-is_py26 = not is_py3 and sys.version_info[1] < 7
 py_version = int(str(sys.version_info[0]) + str(sys.version_info[1]))
 
 
@@ -35,28 +32,36 @@ class DummyFile(object):
         del self.loader
 
 
-def find_module_py34(string, path=None, fullname=None):
-    implicit_namespace_pkg = False
+def find_module_py34(string, path=None, full_name=None, is_global_search=True):
     spec = None
     loader = None
 
-    spec = importlib.machinery.PathFinder.find_spec(string, path)
-    if hasattr(spec, 'origin'):
-        origin = spec.origin
-        implicit_namespace_pkg = origin == 'namespace'
+    for finder in sys.meta_path:
+        if is_global_search and finder != importlib.machinery.PathFinder:
+            p = None
+        else:
+            p = path
+        try:
+            find_spec = finder.find_spec
+        except AttributeError:
+            # These are old-school clases that still have a different API, just
+            # ignore those.
+            continue
 
-    # We try to disambiguate implicit namespace pkgs with non implicit namespace pkgs
-    if implicit_namespace_pkg:
-        fullname = string if not path else fullname
-        implicit_ns_info = ImplicitNSInfo(fullname, spec.submodule_search_locations._path)
-        return None, implicit_ns_info, False
+        spec = find_spec(string, p)
+        if spec is not None:
+            loader = spec.loader
+            if loader is None and not spec.has_location:
+                # This is a namespace package.
+                full_name = string if not path else full_name
+                implicit_ns_info = ImplicitNSInfo(full_name, spec.submodule_search_locations._path)
+                return None, implicit_ns_info, False
+            break
 
-    # we have found the tail end of the dotted path
-    if hasattr(spec, 'loader'):
-        loader = spec.loader
     return find_module_py33(string, path, loader)
 
-def find_module_py33(string, path=None, loader=None, fullname=None):
+
+def find_module_py33(string, path=None, loader=None, full_name=None, is_global_search=True):
     loader = loader or importlib.machinery.PathFinder.find_module(string, path)
 
     if loader is None and path is None:  # Fallback to find builtins
@@ -74,7 +79,7 @@ def find_module_py33(string, path=None, loader=None, fullname=None):
             raise ImportError("Originally  " + repr(e))
 
     if loader is None:
-        raise ImportError("Couldn't find a loader for {0}".format(string))
+        raise ImportError("Couldn't find a loader for {}".format(string))
 
     try:
         is_package = loader.is_package(string)
@@ -109,7 +114,10 @@ def find_module_py33(string, path=None, loader=None, fullname=None):
     return module_file, module_path, is_package
 
 
-def find_module_pre_py33(string, path=None, fullname=None):
+def find_module_pre_py34(string, path=None, full_name=None, is_global_search=True):
+    # This import is here, because in other places it will raise a
+    # DeprecationWarning.
+    import imp
     try:
         module_file, module_path, description = imp.find_module(string, path)
         module_type = description[2]
@@ -127,14 +135,7 @@ def find_module_pre_py33(string, path=None, fullname=None):
                 if loader:
                     is_package = loader.is_package(string)
                     is_archive = hasattr(loader, 'archive')
-                    try:
-                        module_path = loader.get_filename(string)
-                    except AttributeError:
-                        # fallback for py26
-                        try:
-                            module_path = loader._get_filename(string)
-                        except AttributeError:
-                            continue
+                    module_path = loader.get_filename(string)
                     if is_package:
                         module_path = os.path.dirname(module_path)
                     if is_archive:
@@ -142,14 +143,13 @@ def find_module_pre_py33(string, path=None, fullname=None):
                     file = None
                     if not is_package or is_archive:
                         file = DummyFile(loader, string)
-                    return (file, module_path, is_package)
+                    return file, module_path, is_package
             except ImportError:
                 pass
-    raise ImportError("No module named {0}".format(string))
+    raise ImportError("No module named {}".format(string))
 
 
-find_module = find_module_py33 if is_py33 else find_module_pre_py33
-find_module = find_module_py34 if is_py34  else find_module
+find_module = find_module_py34 if is_py3 else find_module_pre_py34
 find_module.__doc__ = """
 Provides information about a module.
 
@@ -161,12 +161,81 @@ if the module is contained in a package.
 """
 
 
+def _iter_modules(paths, prefix=''):
+    # Copy of pkgutil.iter_modules adapted to work with namespaces
+
+    for path in paths:
+        importer = pkgutil.get_importer(path)
+
+        if not isinstance(importer, importlib.machinery.FileFinder):
+            # We're only modifying the case for FileFinder. All the other cases
+            # still need to be checked (like zip-importing). Do this by just
+            # calling the pkgutil version.
+            for mod_info in pkgutil.iter_modules([path], prefix):
+                yield mod_info
+            continue
+
+        # START COPY OF pkutils._iter_file_finder_modules.
+        if importer.path is None or not os.path.isdir(importer.path):
+            return
+
+        yielded = {}
+
+        try:
+            filenames = os.listdir(importer.path)
+        except OSError:
+            # ignore unreadable directories like import does
+            filenames = []
+        filenames.sort()  # handle packages before same-named modules
+
+        for fn in filenames:
+            modname = inspect.getmodulename(fn)
+            if modname == '__init__' or modname in yielded:
+                continue
+
+            # jedi addition: Avoid traversing special directories
+            if fn.startswith('.') or fn == '__pycache__':
+                continue
+
+            path = os.path.join(importer.path, fn)
+            ispkg = False
+
+            if not modname and os.path.isdir(path) and '.' not in fn:
+                modname = fn
+                # A few jedi modifications: Don't check if there's an
+                # __init__.py
+                try:
+                    os.listdir(path)
+                except OSError:
+                    # ignore unreadable directories like import does
+                    continue
+                ispkg = True
+
+            if modname and '.' not in modname:
+                yielded[modname] = 1
+                yield importer, prefix + modname, ispkg
+        # END COPY
+
+
+iter_modules = _iter_modules if py_version >= 34 else pkgutil.iter_modules
+
+
 class ImplicitNSInfo(object):
     """Stores information returned from an implicit namespace spec"""
     def __init__(self, name, paths):
         self.name = name
         self.paths = paths
 
+
+if is_py3:
+    all_suffixes = importlib.machinery.all_suffixes
+else:
+    def all_suffixes():
+        # Is deprecated and raises a warning in Python 3.6.
+        import imp
+        return [suffix for suffix, _, _ in imp.get_suffixes()]
+
+
 # unicode function
 try:
     unicode = unicode
@@ -174,14 +243,6 @@ except NameError:
     unicode = str
 
 
-# exec function
-if is_py3:
-    def exec_function(source, global_map):
-        exec(source, global_map)
-else:
-    eval(compile("""def exec_function(source, global_map):
-                        exec source in global_map """, 'blub', 'exec'))
-
 # re-raise function
 if is_py3:
     def reraise(exception, traceback):
@@ -201,6 +262,7 @@ Usage::
 
 """
 
+
 class Python3Method(object):
     def __init__(self, func):
         self.func = func
@@ -216,7 +278,7 @@ def use_metaclass(meta, *bases):
     """ Create a class with a metaclass. """
     if not bases:
         bases = (object,)
-    return meta("HackClass", bases, {})
+    return meta("Py2CompatibilityMetaClass", bases, {})
 
 
 try:
@@ -227,47 +289,70 @@ except AttributeError:
     encoding = 'ascii'
 
 
-def u(string):
+def u(string, errors='strict'):
     """Cast to unicode DAMMIT!
     Written because Python2 repr always implicitly casts to a string, so we
     have to cast back to a unicode (and we now that we always deal with valid
     unicode, because we check that in the beginning).
     """
-    if is_py3:
-        return str(string)
-
-    if not isinstance(string, unicode):
-        return unicode(str(string), 'UTF-8')
+    if isinstance(string, bytes):
+        return unicode(string, encoding='UTF-8', errors=errors)
     return string
 
+
+def cast_path(obj):
+    """
+    Take a bytes or str path and cast it to unicode.
+
+    Apparently it is perfectly fine to pass both byte and unicode objects into
+    the sys.path. This probably means that byte paths are normal at other
+    places as well.
+
+    Since this just really complicates everything and Python 2.7 will be EOL
+    soon anyway, just go with always strings.
+    """
+    return u(obj, errors='replace')
+
+
+def force_unicode(obj):
+    # Intentionally don't mix those two up, because those two code paths might
+    # be different in the future (maybe windows?).
+    return cast_path(obj)
+
+
 try:
     import builtins  # module name in python 3
 except ImportError:
-    import __builtin__ as builtins
+    import __builtin__ as builtins  # noqa: F401
 
 
-import ast
+import ast  # noqa: F401
 
 
 def literal_eval(string):
-    # py3.0, py3.1 and py32 don't support unicode literals. Support those, I
-    # don't want to write two versions of the tokenizer.
-    if is_py3 and sys.version_info.minor < 3:
-        if re.match('[uU][\'"]', string):
-            string = string[1:]
     return ast.literal_eval(string)
 
 
 try:
     from itertools import zip_longest
 except ImportError:
-    from itertools import izip_longest as zip_longest  # Python 2
+    from itertools import izip_longest as zip_longest  # Python 2  # noqa: F401
 
 try:
     FileNotFoundError = FileNotFoundError
 except NameError:
     FileNotFoundError = IOError
 
+try:
+    NotADirectoryError = NotADirectoryError
+except NameError:
+    NotADirectoryError = IOError
+
+try:
+    PermissionError = PermissionError
+except NameError:
+    PermissionError = IOError
+
 
 def no_unicode_pprint(dct):
     """
@@ -281,6 +366,14 @@ def no_unicode_pprint(dct):
     print(re.sub("u'", "'", s))
 
 
+def print_to_stderr(*args):
+    if is_py3:
+        eval("print(*args, file=sys.stderr)")
+    else:
+        print >> sys.stderr, args
+    sys.stderr.flush()
+
+
 def utf8_repr(func):
     """
     ``__repr__`` methods in Python 2 don't allow unicode objects to be
@@ -297,3 +390,205 @@ def utf8_repr(func):
         return func
     else:
         return wrapper
+
+
+if is_py3:
+    import queue
+else:
+    import Queue as queue  # noqa: F401
+
+try:
+    # Attempt to load the C implementation of pickle on Python 2 as it is way
+    # faster.
+    import cPickle as pickle
+except ImportError:
+    import pickle
+if sys.version_info[:2] == (3, 3):
+    """
+    Monkeypatch the unpickler in Python 3.3. This is needed, because the
+    argument `encoding='bytes'` is not supported in 3.3, but badly needed to
+    communicate with Python 2.
+    """
+
+    class NewUnpickler(pickle._Unpickler):
+        dispatch = dict(pickle._Unpickler.dispatch)
+
+        def _decode_string(self, value):
+            # Used to allow strings from Python 2 to be decoded either as
+            # bytes or Unicode strings.  This should be used only with the
+            # STRING, BINSTRING and SHORT_BINSTRING opcodes.
+            if self.encoding == "bytes":
+                return value
+            else:
+                return value.decode(self.encoding, self.errors)
+
+        def load_string(self):
+            data = self.readline()[:-1]
+            # Strip outermost quotes
+            if len(data) >= 2 and data[0] == data[-1] and data[0] in b'"\'':
+                data = data[1:-1]
+            else:
+                raise pickle.UnpicklingError("the STRING opcode argument must be quoted")
+            self.append(self._decode_string(pickle.codecs.escape_decode(data)[0]))
+        dispatch[pickle.STRING[0]] = load_string
+
+        def load_binstring(self):
+            # Deprecated BINSTRING uses signed 32-bit length
+            len, = pickle.struct.unpack('<i', self.read(4))
+            if len < 0:
+                raise pickle.UnpicklingError("BINSTRING pickle has negative byte count")
+            data = self.read(len)
+            self.append(self._decode_string(data))
+        dispatch[pickle.BINSTRING[0]] = load_binstring
+
+        def load_short_binstring(self):
+            len = self.read(1)[0]
+            data = self.read(len)
+            self.append(self._decode_string(data))
+        dispatch[pickle.SHORT_BINSTRING[0]] = load_short_binstring
+
+    def load(file, fix_imports=True, encoding="ASCII", errors="strict"):
+        return NewUnpickler(file, fix_imports=fix_imports,
+                            encoding=encoding, errors=errors).load()
+
+    def loads(s, fix_imports=True, encoding="ASCII", errors="strict"):
+        if isinstance(s, str):
+            raise TypeError("Can't load pickle from unicode string")
+        file = pickle.io.BytesIO(s)
+        return NewUnpickler(file, fix_imports=fix_imports,
+                            encoding=encoding, errors=errors).load()
+
+    pickle.Unpickler = NewUnpickler
+    pickle.load = load
+    pickle.loads = loads
+
+
+def pickle_load(file):
+    try:
+        if is_py3:
+            return pickle.load(file, encoding='bytes')
+        return pickle.load(file)
+    # Python on Windows don't throw EOF errors for pipes. So reraise them with
+    # the correct type, which is caught upwards.
+    except OSError:
+        if sys.platform == 'win32':
+            raise EOFError()
+        raise
+
+
+def pickle_dump(data, file, protocol):
+    try:
+        pickle.dump(data, file, protocol)
+        # On Python 3.3 flush throws sometimes an error even though the writing
+        # operation should be completed.
+        file.flush()
+    # Python on Windows don't throw EPIPE errors for pipes. So reraise them with
+    # the correct type and error number.
+    except OSError:
+        if sys.platform == 'win32':
+            raise IOError(errno.EPIPE, "Broken pipe")
+        raise
+
+
+# Determine the highest protocol version compatible for a given list of Python
+# versions.
+def highest_pickle_protocol(python_versions):
+    protocol = 4
+    for version in python_versions:
+        if version[0] == 2:
+            # The minimum protocol version for the versions of Python that we
+            # support (2.7 and 3.3+) is 2.
+            return 2
+        if version[1] < 4:
+            protocol = 3
+    return protocol
+
+
+try:
+    from inspect import Parameter
+except ImportError:
+    class Parameter(object):
+        POSITIONAL_ONLY = object()
+        POSITIONAL_OR_KEYWORD = object()
+        VAR_POSITIONAL = object()
+        KEYWORD_ONLY = object()
+        VAR_KEYWORD = object()
+
+
+class GeneralizedPopen(subprocess.Popen):
+    def __init__(self, *args, **kwargs):
+        if os.name == 'nt':
+            try:
+                # Was introduced in Python 3.7.
+                CREATE_NO_WINDOW = subprocess.CREATE_NO_WINDOW
+            except AttributeError:
+                CREATE_NO_WINDOW = 0x08000000
+            kwargs['creationflags'] = CREATE_NO_WINDOW
+        # The child process doesn't need file descriptors except 0, 1, 2.
+        # This is unix only.
+        kwargs['close_fds'] = 'posix' in sys.builtin_module_names
+        super(GeneralizedPopen, self).__init__(*args, **kwargs)
+
+
+# shutil.which is not available on Python 2.7.
+def which(cmd, mode=os.F_OK | os.X_OK, path=None):
+    """Given a command, mode, and a PATH string, return the path which
+    conforms to the given mode on the PATH, or None if there is no such
+    file.
+
+    `mode` defaults to os.F_OK | os.X_OK. `path` defaults to the result
+    of os.environ.get("PATH"), or can be overridden with a custom search
+    path.
+
+    """
+    # Check that a given file can be accessed with the correct mode.
+    # Additionally check that `file` is not a directory, as on Windows
+    # directories pass the os.access check.
+    def _access_check(fn, mode):
+        return (os.path.exists(fn) and os.access(fn, mode)
+                and not os.path.isdir(fn))
+
+    # If we're given a path with a directory part, look it up directly rather
+    # than referring to PATH directories. This includes checking relative to the
+    # current directory, e.g. ./script
+    if os.path.dirname(cmd):
+        if _access_check(cmd, mode):
+            return cmd
+        return None
+
+    if path is None:
+        path = os.environ.get("PATH", os.defpath)
+    if not path:
+        return None
+    path = path.split(os.pathsep)
+
+    if sys.platform == "win32":
+        # The current directory takes precedence on Windows.
+        if os.curdir not in path:
+            path.insert(0, os.curdir)
+
+        # PATHEXT is necessary to check on Windows.
+        pathext = os.environ.get("PATHEXT", "").split(os.pathsep)
+        # See if the given file matches any of the expected path extensions.
+        # This will allow us to short circuit when given "python.exe".
+        # If it does match, only test that one, otherwise we have to try
+        # others.
+        if any(cmd.lower().endswith(ext.lower()) for ext in pathext):
+            files = [cmd]
+        else:
+            files = [cmd + ext for ext in pathext]
+    else:
+        # On other platforms you don't have things like PATHEXT to tell you
+        # what file suffixes are executable, so just pass on cmd as-is.
+        files = [cmd]
+
+    seen = set()
+    for dir in path:
+        normdir = os.path.normcase(dir)
+        if normdir not in seen:
+            seen.add(normdir)
+            for thefile in files:
+                name = os.path.join(dir, thefile)
+                if _access_check(name, mode):
+                    return name
+    return None

jedi/api/__init__.py

@@ -11,28 +11,32 @@ arguments.
 """
 import os
 import sys
+import warnings
 
 import parso
 from parso.python import tree
-from parso import python_bytes_to_unicode, split_lines
 
-from jedi.parser_utils import get_executable_nodes, get_statement_of_position
+from jedi._compatibility import force_unicode, is_py3
+from jedi.parser_utils import get_executable_nodes
 from jedi import debug
 from jedi import settings
 from jedi import cache
 from jedi.api import classes
 from jedi.api import interpreter
-from jedi.api import usages
 from jedi.api import helpers
 from jedi.api.completion import Completion
+from jedi.api.environment import InterpreterEnvironment
+from jedi.api.project import get_default_project
 from jedi.evaluate import Evaluator
-from jedi.evaluate import representation as er
 from jedi.evaluate import imports
-from jedi.evaluate.param import try_iter_content
+from jedi.evaluate import usages
+from jedi.evaluate.arguments import try_iter_content
 from jedi.evaluate.helpers import get_module_names, evaluate_call_of_leaf
-from jedi.evaluate.sys_path import get_venv_path, dotted_path_in_sys_path
-from jedi.evaluate.iterable import unpack_tuple_to_dict
-from jedi.evaluate.filters import TreeNameDefinition
+from jedi.evaluate.sys_path import dotted_path_in_sys_path
+from jedi.evaluate.filters import TreeNameDefinition, ParamName
+from jedi.evaluate.syntax_tree import tree_name_to_contexts
+from jedi.evaluate.context import ModuleContext
+from jedi.evaluate.context.iterable import unpack_tuple_to_dict
 
 # Jedi uses lots and lots of recursion. By setting this a little bit higher, we
 # can remove some "maximum recursion depth" errors.
@@ -71,15 +75,13 @@ class Script(object):
     :param encoding: The encoding of ``source``, if it is not a
         ``unicode`` object (default ``'utf-8'``).
     :type encoding: str
-    :param source_encoding: The encoding of ``source``, if it is not a
-        ``unicode`` object (default ``'utf-8'``).
-    :type encoding: str
     :param sys_path: ``sys.path`` to use during analysis of the script
     :type sys_path: list
-
+    :param environment: TODO
+    :type sys_path: Environment
     """
     def __init__(self, source=None, line=None, column=None, path=None,
-                 encoding='utf-8', sys_path=None):
+                 encoding='utf-8', sys_path=None, environment=None):
         self._orig_path = path
         # An empty path (also empty string) should always result in no path.
         self.path = os.path.abspath(path) if path else None
@@ -89,57 +91,77 @@ class Script(object):
             with open(path, 'rb') as f:
                 source = f.read()
 
-        # TODO do we really want that?
-        self._source = python_bytes_to_unicode(source, encoding, errors='replace')
-        self._code_lines = split_lines(self._source)
+        # Load the Python grammar of the current interpreter.
+        self._grammar = parso.load_grammar()
+
+        if sys_path is not None and not is_py3:
+            sys_path = list(map(force_unicode, sys_path))
+
+        # Load the Python grammar of the current interpreter.
+        project = get_default_project(
+            os.path.dirname(self.path)if path else os.getcwd()
+        )
+        # TODO deprecate and remove sys_path from the Script API.
+        if sys_path is not None:
+            project._sys_path = sys_path
+        self._evaluator = Evaluator(
+            project, environment=environment, script_path=self.path
+        )
+        self._project = project
+        debug.speed('init')
+        self._module_node, source = self._evaluator.parse_and_get_code(
+            code=source,
+            path=self.path,
+            encoding=encoding,
+            cache=False,  # No disk cache, because the current script often changes.
+            diff_cache=settings.fast_parser,
+            cache_path=settings.cache_directory,
+        )
+        debug.speed('parsed')
+        self._code_lines = parso.split_lines(source, keepends=True)
+        self._code = source
         line = max(len(self._code_lines), 1) if line is None else line
         if not (0 < line <= len(self._code_lines)):
             raise ValueError('`line` parameter is not in a valid range.')
 
-        line_len = len(self._code_lines[line - 1])
+        line_string = self._code_lines[line - 1]
+        line_len = len(line_string)
+        if line_string.endswith('\r\n'):
+            line_len -= 1
+        if line_string.endswith('\n'):
+            line_len -= 1
+
         column = line_len if column is None else column
         if not (0 <= column <= line_len):
-            raise ValueError('`column` parameter is not in a valid range.')
+            raise ValueError('`column` parameter (%d) is not in a valid range '
+                             '(0-%d) for line %d (%r).' % (
+                                 column, line_len, line, line_string))
         self._pos = line, column
         self._path = path
 
         cache.clear_time_caches()
         debug.reset_time()
 
-        # Load the Python grammar of the current interpreter.
-        self._grammar = parso.load_grammar()
-        if sys_path is None:
-            venv = os.getenv('VIRTUAL_ENV')
-            if venv:
-                sys_path = list(get_venv_path(venv))
-        self._evaluator = Evaluator(self._grammar, sys_path=sys_path)
-        debug.speed('init')
-
-    @cache.memoize_method
-    def _get_module_node(self):
-        return self._grammar.parse(
-            code=self._source,
-            path=self.path,
-            cache=False,  # No disk cache, because the current script often changes.
-            diff_cache=True,
-            cache_path=settings.cache_directory
-        )
-
-    @cache.memoize_method
     def _get_module(self):
-        module = er.ModuleContext(
-            self._evaluator,
-            self._get_module_node(),
-            self.path
-        )
+        name = '__main__'
         if self.path is not None:
-            name = dotted_path_in_sys_path(self._evaluator.sys_path, self.path)
-            if name is not None:
-                imports.add_module(self._evaluator, name, module)
+            import_names = dotted_path_in_sys_path(self._evaluator.get_sys_path(), self.path)
+            if import_names is not None:
+                name = '.'.join(import_names)
+
+        module = ModuleContext(
+            self._evaluator, self._module_node, self.path,
+            code_lines=self._code_lines
+        )
+        imports.add_module_to_cache(self._evaluator, name, module)
         return module
 
     def __repr__(self):
-        return '<%s: %s>' % (self.__class__.__name__, repr(self._orig_path))
+        return '<%s: %s %r>' % (
+            self.__class__.__name__,
+            repr(self._orig_path),
+            self._evaluator.environment,
+        )
 
     def completions(self):
         """
@@ -155,6 +177,24 @@ class Script(object):
             self._pos, self.call_signatures
         )
         completions = completion.completions()
+
+        def iter_import_completions():
+            for c in completions:
+                tree_name = c._name.tree_name
+                if tree_name is None:
+                    continue
+                definition = tree_name.get_definition()
+                if definition is not None \
+                        and definition.type in ('import_name', 'import_from'):
+                    yield c
+
+        if len(list(iter_import_completions())) > 10:
+            # For now disable completions if there's a lot of imports that
+            # might potentially be resolved. This is the case for tensorflow
+            # and has been fixed for it. This is obviously temporary until we
+            # have a better solution.
+            self._evaluator.infer_enabled = False
+
         debug.speed('completions end')
         return completions
 
@@ -170,10 +210,9 @@ class Script(object):
 
         :rtype: list of :class:`classes.Definition`
         """
-        module_node = self._get_module_node()
-        leaf = module_node.get_name_of_position(self._pos)
+        leaf = self._module_node.get_name_of_position(self._pos)
         if leaf is None:
-            leaf = module_node.get_leaf_for_position(self._pos)
+            leaf = self._module_node.get_leaf_for_position(self._pos)
             if leaf is None:
                 return []
 
@@ -187,29 +226,45 @@ class Script(object):
         # the API.
         return helpers.sorted_definitions(set(defs))
 
-    def goto_assignments(self, follow_imports=False):
+    def goto_assignments(self, follow_imports=False, follow_builtin_imports=False):
         """
         Return the first definition found, while optionally following imports.
         Multiple objects may be returned, because Python itself is a
         dynamic language, which means depending on an option you can have two
         different versions of a function.
 
+        :param follow_imports: The goto call will follow imports.
+        :param follow_builtin_imports: If follow_imports is True will decide if
+            it follow builtin imports.
         :rtype: list of :class:`classes.Definition`
         """
         def filter_follow_imports(names, check):
             for name in names:
                 if check(name):
-                    for result in filter_follow_imports(name.goto(), check):
-                        yield result
+                    new_names = list(filter_follow_imports(name.goto(), check))
+                    found_builtin = False
+                    if follow_builtin_imports:
+                        for new_name in new_names:
+                            if new_name.start_pos is None:
+                                found_builtin = True
+
+                    if found_builtin and not isinstance(name, imports.SubModuleName):
+                        yield name
+                    else:
+                        for new_name in new_names:
+                            yield new_name
                 else:
                     yield name
 
-        names = self._goto()
+        tree_name = self._module_node.get_name_of_position(self._pos)
+        if tree_name is None:
+            return []
+        context = self._evaluator.create_context(self._get_module(), tree_name)
+        names = list(self._evaluator.goto(context, tree_name))
+
         if follow_imports:
             def check(name):
-                if isinstance(name, er.ModuleName):
-                    return False
-                return name.api_type == 'module'
+                return name.is_import()
         else:
             def check(name):
                 return isinstance(name, imports.SubModuleName)
@@ -219,17 +274,7 @@ class Script(object):
         defs = [classes.Definition(self._evaluator, d) for d in set(names)]
         return helpers.sorted_definitions(defs)
 
-    def _goto(self):
-        """
-        Used for goto_assignments and usages.
-        """
-        name = self._get_module_node().get_name_of_position(self._pos)
-        if name is None:
-            return []
-        context = self._evaluator.create_context(self._get_module(), name)
-        return list(self._evaluator.goto(context, name))
-
-    def usages(self, additional_module_paths=()):
+    def usages(self, additional_module_paths=(), **kwargs):
         """
         Return :class:`classes.Definition` objects, which contain all
         names that point to the definition of the name under the cursor. This
@@ -238,38 +283,31 @@ class Script(object):
 
         .. todo:: Implement additional_module_paths
 
+        :param additional_module_paths: Deprecated, never ever worked.
+        :param include_builtins: Default True, checks if a usage is a builtin
+            (e.g. ``sys``) and in that case does not return it.
         :rtype: list of :class:`classes.Definition`
         """
-        temp, settings.dynamic_flow_information = \
-            settings.dynamic_flow_information, False
-        try:
-            module_node = self._get_module_node()
-            user_stmt = get_statement_of_position(module_node, self._pos)
-            definition_names = self._goto()
-            if not definition_names and isinstance(user_stmt, tree.Import):
-                # For not defined imports (goto doesn't find something, we take
-                # the name as a definition. This is enough, because every name
-                # points to it.
-                name = user_stmt.get_name_of_position(self._pos)
-                if name is None:
-                    # Must be syntax
-                    return []
-                definition_names = [TreeNameDefinition(self._get_module(), name)]
+        if additional_module_paths:
+            warnings.warn(
+                "Deprecated since version 0.12.0. This never even worked, just ignore it.",
+                DeprecationWarning,
+                stacklevel=2
+            )
 
-            if not definition_names:
-                # Without a definition for a name we cannot find references.
+        def _usages(include_builtins=True):
+            tree_name = self._module_node.get_name_of_position(self._pos)
+            if tree_name is None:
+                # Must be syntax
                 return []
 
-            definition_names = usages.resolve_potential_imports(self._evaluator,
-                                                                definition_names)
+            names = usages.usages(self._get_module(), tree_name)
 
-            modules = set([d.get_root_context() for d in definition_names])
-            modules.add(self._get_module())
-            definitions = usages.usages(self._evaluator, definition_names, modules)
-        finally:
-            settings.dynamic_flow_information = temp
-
-        return helpers.sorted_definitions(set(definitions))
+            definitions = [classes.Definition(self._evaluator, n) for n in names]
+            if not include_builtins:
+                definitions = [d for d in definitions if not d.in_builtin_module()]
+            return helpers.sorted_definitions(definitions)
+        return _usages(**kwargs)
 
     def call_signatures(self):
         """
@@ -288,7 +326,7 @@ class Script(object):
         :rtype: list of :class:`classes.CallSignature`
         """
         call_signature_details = \
-            helpers.get_call_signature_details(self._get_module_node(), self._pos)
+            helpers.get_call_signature_details(self._module_node, self._pos)
         if call_signature_details is None:
             return []
 
@@ -313,16 +351,14 @@ class Script(object):
 
     def _analysis(self):
         self._evaluator.is_analysis = True
-        module_node = self._get_module_node()
-        self._evaluator.analysis_modules = [module_node]
+        self._evaluator.analysis_modules = [self._module_node]
+        module = self._get_module()
         try:
-            for node in get_executable_nodes(module_node):
-                context = self._get_module().create_context(node)
+            for node in get_executable_nodes(self._module_node):
+                context = module.create_context(node)
                 if node.type in ('funcdef', 'classdef'):
-                    # TODO This is stupid, should be private
-                    from jedi.evaluate.finder import _name_to_types
                     # Resolve the decorators.
-                    _name_to_types(self._evaluator, context, node.children[1])
+                    tree_name_to_contexts(self._evaluator, context, node.children[1])
                 elif isinstance(node, tree.Import):
                     import_names = set(node.get_defined_names())
                     if node.is_nested():
@@ -383,21 +419,28 @@ class Interpreter(Script):
         except Exception:
             raise TypeError("namespaces must be a non-empty list of dicts.")
 
-        super(Interpreter, self).__init__(source, **kwds)
+        environment = kwds.get('environment', None)
+        if environment is None:
+            environment = InterpreterEnvironment()
+        else:
+            if not isinstance(environment, InterpreterEnvironment):
+                raise TypeError("The environment needs to be an InterpreterEnvironment subclass.")
+
+        super(Interpreter, self).__init__(source, environment=environment, **kwds)
         self.namespaces = namespaces
 
     def _get_module(self):
-        parser_module = super(Interpreter, self)._get_module_node()
         return interpreter.MixedModuleContext(
             self._evaluator,
-            parser_module,
+            self._module_node,
             self.namespaces,
-            path=self.path
+            path=self.path,
+            code_lines=self._code_lines,
         )
 
 
 def names(source=None, path=None, encoding='utf-8', all_scopes=False,
-          definitions=True, references=False):
+          definitions=True, references=False, environment=None):
     """
     Returns a list of `Definition` objects, containing name parts.
     This means you can call ``Definition.goto_assignments()`` and get the
@@ -416,17 +459,25 @@ def names(source=None, path=None, encoding='utf-8', all_scopes=False,
         is_def = _def._name.tree_name.is_definition()
         return definitions and is_def or references and not is_def
 
+    def create_name(name):
+        if name.parent.type == 'param':
+            cls = ParamName
+        else:
+            cls = TreeNameDefinition
+        is_module = name.parent.type == 'file_input'
+        return cls(
+            module_context.create_context(name if is_module else name.parent),
+            name
+        )
+
     # Set line/column to a random position, because they don't matter.
-    script = Script(source, line=1, column=0, path=path, encoding=encoding)
+    script = Script(source, line=1, column=0, path=path, encoding=encoding, environment=environment)
     module_context = script._get_module()
     defs = [
         classes.Definition(
             script._evaluator,
-            TreeNameDefinition(
-                module_context.create_context(name if name.parent.type == 'file_input' else name.parent),
-                name
-            )
-        ) for name in get_module_names(script._get_module_node(), all_scopes)
+            create_name(name)
+        ) for name in get_module_names(script._module_node, all_scopes)
     ]
     return sorted(filter(def_ref_filter, defs), key=lambda x: (x.line, x.column))
 

jedi/api/classes.py

@@ -5,19 +5,16 @@ the interesting information about completion and goto operations.
 """
 import re
 
-from parso.cache import parser_cache
 from parso.python.tree import search_ancestor
 
-from jedi._compatibility import u
 from jedi import settings
-from jedi import common
+from jedi.evaluate.utils import ignored, unite
 from jedi.cache import memoize_method
-from jedi.evaluate import representation as er
-from jedi.evaluate import instance
 from jedi.evaluate import imports
 from jedi.evaluate import compiled
-from jedi.evaluate.filters import ParamName
 from jedi.evaluate.imports import ImportName
+from jedi.evaluate.context import instance
+from jedi.evaluate.context import ClassContext, FunctionExecutionContext
 from jedi.api.keywords import KeywordName
 
 
@@ -125,13 +122,14 @@ class BaseDefinition(object):
 
         Finally, here is what you can get from :attr:`type`:
 
-        >>> defs[0].type
+        >>> defs = [str(d.type) for d in defs]  # It's unicode and in Py2 has u before it.
+        >>> defs[0]
         'module'
-        >>> defs[1].type
+        >>> defs[1]
         'class'
-        >>> defs[2].type
+        >>> defs[2]
         'instance'
-        >>> defs[3].type
+        >>> defs[3]
         'function'
 
         """
@@ -159,7 +157,7 @@ class BaseDefinition(object):
                 except IndexError:
                     pass
 
-            if name.api_type == 'module':
+            if name.api_type in 'module':
                 module_contexts = name.infer()
                 if module_contexts:
                     module_context, = module_contexts
@@ -259,7 +257,7 @@ class BaseDefinition(object):
     @property
     def description(self):
         """A textual description of the object."""
-        return u(self._name.string_name)
+        return self._name.string_name
 
     @property
     def full_name(self):
@@ -290,7 +288,7 @@ class BaseDefinition(object):
         if not path:
             return None  # for keywords the path is empty
 
-        with common.ignored(KeyError):
+        with ignored(KeyError):
             path[0] = self._mapping[path[0]]
         for key, repl in self._tuple_mapping.items():
             if tuple(path[:len(key)]) == key:
@@ -322,11 +320,11 @@ class BaseDefinition(object):
                 param_names = list(context.get_param_names())
                 if isinstance(context, instance.BoundMethod):
                     param_names = param_names[1:]
-            elif isinstance(context, (instance.AbstractInstanceContext, er.ClassContext)):
-                if isinstance(context, er.ClassContext):
-                    search = '__init__'
+            elif isinstance(context, (instance.AbstractInstanceContext, ClassContext)):
+                if isinstance(context, ClassContext):
+                    search = u'__init__'
                 else:
-                    search = '__call__'
+                    search = u'__call__'
                 names = context.get_function_slot_names(search)
                 if not names:
                     return []
@@ -335,7 +333,7 @@ class BaseDefinition(object):
                 # there's no better solution.
                 inferred = names[0].infer()
                 param_names = get_param_names(next(iter(inferred)))
-                if isinstance(context, er.ClassContext):
+                if isinstance(context, ClassContext):
                     param_names = param_names[1:]
                 return param_names
             elif isinstance(context, compiled.CompiledObject):
@@ -344,7 +342,7 @@ class BaseDefinition(object):
 
         followed = list(self._name.infer())
         if not followed or not hasattr(followed[0], 'py__call__'):
-            raise AttributeError()
+            raise AttributeError('There are no params defined on this.')
         context = followed[0]  # only check the first one.
 
         return [Definition(self._evaluator, n) for n in get_param_names(context)]
@@ -354,11 +352,8 @@ class BaseDefinition(object):
         if context is None:
             return None
 
-        if isinstance(context, er.FunctionExecutionContext):
-            # TODO the function context should be a part of the function
-            # execution context.
-            context = er.FunctionContext(
-                self._evaluator, context.parent_context, context.tree_node)
+        if isinstance(context, FunctionExecutionContext):
+            context = context.function_context
         return Definition(self._evaluator, context.name)
 
     def __repr__(self):
@@ -377,8 +372,7 @@ class BaseDefinition(object):
         if self.in_builtin_module():
             return ''
 
-        path = self._name.get_root_context().py__file__()
-        lines = parser_cache[self._evaluator.grammar._hashed][path].lines
+        lines = self._name.get_root_context().code_lines
 
         index = self._name.start_pos[0] - 1
         start_index = max(index - before, 0)
@@ -406,9 +400,10 @@ class Completion(BaseDefinition):
                 and self.type == 'Function':
             append = '('
 
-        if isinstance(self._name, ParamName) and self._stack is not None:
-            node_names = list(self._stack.get_node_names(self._evaluator.grammar._pgen_grammar))
-            if 'trailer' in node_names and 'argument' not in node_names:
+        if self._name.api_type == 'param' and self._stack is not None:
+            nonterminals = [stack_node.nonterminal for stack_node in self._stack]
+            if 'trailer' in nonterminals and 'argument' not in nonterminals:
+                # TODO this doesn't work for nested calls.
                 append += '='
 
         name = self._name.string_name
@@ -525,7 +520,7 @@ class Definition(BaseDefinition):
             if typ == 'function':
                 # For the description we want a short and a pythonic way.
                 typ = 'def'
-            return typ + ' ' + u(self._name.string_name)
+            return typ + ' ' + self._name.string_name
         elif typ == 'param':
             code = search_ancestor(tree_name, 'param').get_code(
                 include_prefix=False,
@@ -533,15 +528,14 @@ class Definition(BaseDefinition):
             )
             return typ + ' ' + code
 
-
         definition = tree_name.get_definition() or tree_name
         # Remove the prefix, because that's not what we want for get_code
         # here.
         txt = definition.get_code(include_prefix=False)
         # Delete comments:
-        txt = re.sub('#[^\n]+\n', ' ', txt)
+        txt = re.sub(r'#[^\n]+\n', ' ', txt)
         # Delete multi spaces/newlines
-        txt = re.sub('\s+', ' ', txt).strip()
+        txt = re.sub(r'\s+', ' ', txt).strip()
         return txt
 
     @property
@@ -555,7 +549,7 @@ class Definition(BaseDefinition):
         .. todo:: Add full path. This function is should return a
             `module.class.function` path.
         """
-        position = '' if self.in_builtin_module else '@%s' % (self.line)
+        position = '' if self.in_builtin_module else '@%s' % self.line
         return "%s:%s%s" % (self.module_name, self.description, position)
 
     @memoize_method
@@ -567,7 +561,7 @@ class Definition(BaseDefinition):
         """
         defs = self._name.infer()
         return sorted(
-            common.unite(defined_names(self._evaluator, d) for d in defs),
+            unite(defined_names(self._evaluator, d) for d in defs),
             key=lambda s: s._name.start_pos or (0, 0)
         )
 
@@ -641,9 +635,18 @@ class CallSignature(Definition):
         """
         return self._bracket_start_pos
 
+    @property
+    def _params_str(self):
+        return ', '.join([p.description[6:]
+                          for p in self.params])
+
     def __repr__(self):
-        return '<%s: %s index %s>' % \
-            (type(self).__name__, self._name.string_name, self.index)
+        return '<%s: %s index=%r params=[%s]>' % (
+            type(self).__name__,
+            self._name.string_name,
+            self._index,
+            self._params_str,
+        )
 
 
 class _Help(object):

jedi/api/completion.py

@@ -1,7 +1,8 @@
-from parso.python import token
+from parso.python.token import PythonTokenTypes
 from parso.python import tree
 from parso.tree import search_ancestor, Leaf
 
+from jedi._compatibility import Parameter
 from jedi import debug
 from jedi import settings
 from jedi.api import classes
@@ -18,24 +19,21 @@ def get_call_signature_param_names(call_signatures):
     for call_sig in call_signatures:
         for p in call_sig.params:
             # Allow protected access, because it's a public API.
-            tree_name = p._name.tree_name
-            # Compiled modules typically don't allow keyword arguments.
-            if tree_name is not None:
-                # Allow access on _definition here, because it's a
-                # public API and we don't want to make the internal
-                # Name object public.
-                tree_param = tree.search_ancestor(tree_name, 'param')
-                if tree_param.star_count == 0:  # no *args/**kwargs
-                    yield p._name
+            if p._name.get_kind() in (Parameter.POSITIONAL_OR_KEYWORD,
+                                      Parameter.KEYWORD_ONLY):
+                yield p._name
 
 
 def filter_names(evaluator, completion_names, stack, like_name):
     comp_dct = {}
+    if settings.case_insensitive_completion:
+        like_name = like_name.lower()
     for name in completion_names:
-        if settings.case_insensitive_completion \
-                and name.string_name.lower().startswith(like_name.lower()) \
-                or name.string_name.startswith(like_name):
+        string = name.string_name
+        if settings.case_insensitive_completion:
+            string = string.lower()
 
+        if string.startswith(like_name):
             new = classes.Completion(
                 evaluator,
                 name,
@@ -59,7 +57,8 @@ def get_user_scope(module_context, position):
         def scan(scope):
             for s in scope.children:
                 if s.start_pos <= position <= s.end_pos:
-                    if isinstance(s, (tree.Scope, tree.Flow)):
+                    if isinstance(s, (tree.Scope, tree.Flow)) \
+                            or s.type in ('async_stmt', 'async_funcdef'):
                         return scan(s) or s
                     elif s.type in ('suite', 'decorated'):
                         return scan(s)
@@ -123,11 +122,11 @@ class Completion:
         grammar = self._evaluator.grammar
 
         try:
-            self.stack = helpers.get_stack_at_position(
+            self.stack = stack = helpers.get_stack_at_position(
                 grammar, self._code_lines, self._module_node, self._position
             )
         except helpers.OnErrorLeaf as e:
-            self.stack = None
+            self.stack = stack = None
             if e.error_leaf.value == '.':
                 # After ErrorLeaf's that are dots, we will not do any
                 # completions since this probably just confuses the user.
@@ -136,10 +135,10 @@ class Completion:
 
             return self._global_completions()
 
-        allowed_keywords, allowed_tokens = \
-            helpers.get_possible_completion_types(grammar._pgen_grammar, self.stack)
+        allowed_transitions = \
+            list(stack._allowed_transition_names_and_token_types())
 
-        if 'if' in allowed_keywords:
+        if 'if' in allowed_transitions:
             leaf = self._module_node.get_leaf_for_position(self._position, include_prefixes=True)
             previous_leaf = leaf.get_previous_leaf()
 
@@ -165,50 +164,52 @@ class Completion:
                     # Compare indents
                     if stmt.start_pos[1] == indent:
                         if type_ == 'if_stmt':
-                            allowed_keywords += ['elif', 'else']
+                            allowed_transitions += ['elif', 'else']
                         elif type_ == 'try_stmt':
-                            allowed_keywords += ['except', 'finally', 'else']
+                            allowed_transitions += ['except', 'finally', 'else']
                         elif type_ == 'for_stmt':
-                            allowed_keywords.append('else')
+                            allowed_transitions.append('else')
 
-        completion_names = list(self._get_keyword_completion_names(allowed_keywords))
+        completion_names = list(self._get_keyword_completion_names(allowed_transitions))
 
-        if token.NAME in allowed_tokens or token.INDENT in allowed_tokens:
+        if any(t in allowed_transitions for t in (PythonTokenTypes.NAME,
+                                                  PythonTokenTypes.INDENT)):
             # This means that we actually have to do type inference.
 
-            symbol_names = list(self.stack.get_node_names(grammar._pgen_grammar))
+            nonterminals = [stack_node.nonterminal for stack_node in stack]
 
-            nodes = list(self.stack.get_nodes())
+            nodes = [node for stack_node in stack for node in stack_node.nodes]
 
             if nodes and nodes[-1] in ('as', 'def', 'class'):
                 # No completions for ``with x as foo`` and ``import x as foo``.
                 # Also true for defining names as a class or function.
                 return list(self._get_class_context_completions(is_function=True))
-            elif "import_stmt" in symbol_names:
-                level, names = self._parse_dotted_names(nodes, "import_from" in symbol_names)
+            elif "import_stmt" in nonterminals:
+                level, names = self._parse_dotted_names(nodes, "import_from" in nonterminals)
 
-                only_modules = not ("import_from" in symbol_names and 'import' in nodes)
+                only_modules = not ("import_from" in nonterminals and 'import' in nodes)
                 completion_names += self._get_importer_names(
                     names,
                     level,
                     only_modules=only_modules,
                 )
-            elif symbol_names[-1] in ('trailer', 'dotted_name') and nodes[-1] == '.':
+            elif nonterminals[-1] in ('trailer', 'dotted_name') and nodes[-1] == '.':
                 dot = self._module_node.get_leaf_for_position(self._position)
                 completion_names += self._trailer_completions(dot.get_previous_leaf())
             else:
                 completion_names += self._global_completions()
                 completion_names += self._get_class_context_completions(is_function=False)
 
-            if 'trailer' in symbol_names:
+            if 'trailer' in nonterminals:
                 call_signatures = self._call_signatures_method()
                 completion_names += get_call_signature_param_names(call_signatures)
 
         return completion_names
 
-    def _get_keyword_completion_names(self, keywords_):
-        for k in keywords_:
-            yield keywords.keyword(self._evaluator, k).name
+    def _get_keyword_completion_names(self, allowed_transitions):
+        for k in allowed_transitions:
+            if isinstance(k, str) and k.isalpha():
+                yield keywords.KeywordName(self._evaluator, k)
 
     def _global_completions(self):
         context = get_user_scope(self._module_context, self._position)

jedi/api/environment.py

@@ -0,0 +1,445 @@
+"""
+Environments are a way to activate different Python versions or Virtualenvs for
+static analysis. The Python binary in that environment is going to be executed.
+"""
+import os
+import sys
+import hashlib
+import filecmp
+from collections import namedtuple
+
+from jedi._compatibility import highest_pickle_protocol, which
+from jedi.cache import memoize_method, time_cache
+from jedi.evaluate.compiled.subprocess import CompiledSubprocess, \
+    EvaluatorSameProcess, EvaluatorSubprocess
+
+import parso
+
+_VersionInfo = namedtuple('VersionInfo', 'major minor micro')
+
+_SUPPORTED_PYTHONS = ['3.7', '3.6', '3.5', '3.4', '3.3', '2.7']
+_SAFE_PATHS = ['/usr/bin', '/usr/local/bin']
+_CURRENT_VERSION = '%s.%s' % (sys.version_info.major, sys.version_info.minor)
+
+
+class InvalidPythonEnvironment(Exception):
+    """
+    If you see this exception, the Python executable or Virtualenv you have
+    been trying to use is probably not a correct Python version.
+    """
+
+
+class _BaseEnvironment(object):
+    @memoize_method
+    def get_grammar(self):
+        version_string = '%s.%s' % (self.version_info.major, self.version_info.minor)
+        return parso.load_grammar(version=version_string)
+
+    @property
+    def _sha256(self):
+        try:
+            return self._hash
+        except AttributeError:
+            self._hash = _calculate_sha256_for_file(self.executable)
+            return self._hash
+
+
+def _get_info():
+    return (
+        sys.executable,
+        sys.prefix,
+        sys.version_info[:3],
+    )
+
+
+class Environment(_BaseEnvironment):
+    """
+    This class is supposed to be created by internal Jedi architecture. You
+    should not create it directly. Please use create_environment or the other
+    functions instead. It is then returned by that function.
+    """
+    _subprocess = None
+
+    def __init__(self, executable):
+        self._start_executable = executable
+        # Initialize the environment
+        self._get_subprocess()
+
+    def _get_subprocess(self):
+        if self._subprocess is not None and not self._subprocess.is_crashed:
+            return self._subprocess
+
+        try:
+            self._subprocess = CompiledSubprocess(self._start_executable)
+            info = self._subprocess._send(None, _get_info)
+        except Exception as exc:
+            raise InvalidPythonEnvironment(
+                "Could not get version information for %r: %r" % (
+                    self._start_executable,
+                    exc))
+
+        # Since it could change and might not be the same(?) as the one given,
+        # set it here.
+        self.executable = info[0]
+        """
+        The Python executable, matches ``sys.executable``.
+        """
+        self.path = info[1]
+        """
+        The path to an environment, matches ``sys.prefix``.
+        """
+        self.version_info = _VersionInfo(*info[2])
+        """
+        Like ``sys.version_info``. A tuple to show the current Environment's
+        Python version.
+        """
+
+        # py2 sends bytes via pickle apparently?!
+        if self.version_info.major == 2:
+            self.executable = self.executable.decode()
+            self.path = self.path.decode()
+
+        # Adjust pickle protocol according to host and client version.
+        self._subprocess._pickle_protocol = highest_pickle_protocol([
+            sys.version_info, self.version_info])
+
+        return self._subprocess
+
+    def __repr__(self):
+        version = '.'.join(str(i) for i in self.version_info)
+        return '<%s: %s in %s>' % (self.__class__.__name__, version, self.path)
+
+    def get_evaluator_subprocess(self, evaluator):
+        return EvaluatorSubprocess(evaluator, self._get_subprocess())
+
+    @memoize_method
+    def get_sys_path(self):
+        """
+        The sys path for this environment. Does not include potential
+        modifications like ``sys.path.append``.
+
+        :returns: list of str
+        """
+        # It's pretty much impossible to generate the sys path without actually
+        # executing Python. The sys path (when starting with -S) itself depends
+        # on how the Python version was compiled (ENV variables).
+        # If you omit -S when starting Python (normal case), additionally
+        # site.py gets executed.
+        return self._get_subprocess().get_sys_path()
+
+
+class SameEnvironment(Environment):
+    def __init__(self):
+        self._start_executable = self.executable = sys.executable
+        self.path = sys.prefix
+        self.version_info = _VersionInfo(*sys.version_info[:3])
+
+
+class InterpreterEnvironment(_BaseEnvironment):
+    def __init__(self):
+        self.version_info = _VersionInfo(*sys.version_info[:3])
+
+    def get_evaluator_subprocess(self, evaluator):
+        return EvaluatorSameProcess(evaluator)
+
+    def get_sys_path(self):
+        return sys.path
+
+
+def _get_virtual_env_from_var():
+    """Get virtualenv environment from VIRTUAL_ENV environment variable.
+
+    It uses `safe=False` with ``create_environment``, because the environment
+    variable is considered to be safe / controlled by the user solely.
+    """
+    var = os.environ.get('VIRTUAL_ENV')
+    if var:
+        if var == sys.prefix:
+            return _try_get_same_env()
+
+        try:
+            return create_environment(var, safe=False)
+        except InvalidPythonEnvironment:
+            pass
+
+
+def _calculate_sha256_for_file(path):
+    sha256 = hashlib.sha256()
+    with open(path, 'rb') as f:
+        for block in iter(lambda: f.read(filecmp.BUFSIZE), b''):
+            sha256.update(block)
+    return sha256.hexdigest()
+
+
+def get_default_environment():
+    """
+    Tries to return an active Virtualenv. If there is no VIRTUAL_ENV variable
+    set it will return the latest Python version installed on the system. This
+    makes it possible to use as many new Python features as possible when using
+    autocompletion and other functionality.
+
+    :returns: :class:`Environment`
+    """
+    virtual_env = _get_virtual_env_from_var()
+    if virtual_env is not None:
+        return virtual_env
+
+    return _try_get_same_env()
+
+
+def _try_get_same_env():
+    env = SameEnvironment()
+    if not os.path.basename(env.executable).lower().startswith('python'):
+        # This tries to counter issues with embedding. In some cases (e.g.
+        # VIM's Python Mac/Windows, sys.executable is /foo/bar/vim. This
+        # happens, because for Mac a function called `_NSGetExecutablePath` is
+        # used and for Windows `GetModuleFileNameW`. These are both platform
+        # specific functions. For all other systems sys.executable should be
+        # alright. However here we try to generalize:
+        #
+        # 1. Check if the executable looks like python (heuristic)
+        # 2. In case it's not try to find the executable
+        # 3. In case we don't find it use an interpreter environment.
+        #
+        # The last option will always work, but leads to potential crashes of
+        # Jedi - which is ok, because it happens very rarely and even less,
+        # because the code below should work for most cases.
+        if os.name == 'nt':
+            # The first case would be a virtualenv and the second a normal
+            # Python installation.
+            checks = (r'Scripts\python.exe', 'python.exe')
+        else:
+            # For unix it looks like Python is always in a bin folder.
+            checks = (
+                'bin/python%s.%s' % (sys.version_info[0], sys.version[1]),
+                'bin/python%s' % (sys.version_info[0]),
+                'bin/python',
+            )
+        for check in checks:
+            guess = os.path.join(sys.exec_prefix, check)
+            if os.path.isfile(guess):
+                # Bingo - We think we have our Python.
+                return Environment(guess)
+        # It looks like there is no reasonable Python to be found.
+        return InterpreterEnvironment()
+    # If no virtualenv is found, use the environment we're already
+    # using.
+    return env
+
+
+def get_cached_default_environment():
+    var = os.environ.get('VIRTUAL_ENV')
+    environment = _get_cached_default_environment()
+    if var and var != environment.path:
+        _get_cached_default_environment.clear_cache()
+        return _get_cached_default_environment()
+    return environment
+
+
+@time_cache(seconds=10 * 60)  # 10 Minutes
+def _get_cached_default_environment():
+    return get_default_environment()
+
+
+def find_virtualenvs(paths=None, **kwargs):
+    """
+    :param paths: A list of paths in your file system to be scanned for
+        Virtualenvs. It will search in these paths and potentially execute the
+        Python binaries. Also the VIRTUAL_ENV variable will be checked if it
+        contains a valid Virtualenv.
+    :param safe: Default True. In case this is False, it will allow this
+        function to execute potential `python` environments. An attacker might
+        be able to drop an executable in a path this function is searching by
+        default. If the executable has not been installed by root, it will not
+        be executed.
+
+    :yields: :class:`Environment`
+    """
+    def py27_comp(paths=None, safe=True):
+        if paths is None:
+            paths = []
+
+        _used_paths = set()
+
+        # Using this variable should be safe, because attackers might be able
+        # to drop files (via git) but not environment variables.
+        virtual_env = _get_virtual_env_from_var()
+        if virtual_env is not None:
+            yield virtual_env
+            _used_paths.add(virtual_env.path)
+
+        for directory in paths:
+            if not os.path.isdir(directory):
+                continue
+
+            directory = os.path.abspath(directory)
+            for path in os.listdir(directory):
+                path = os.path.join(directory, path)
+                if path in _used_paths:
+                    # A path shouldn't be evaluated twice.
+                    continue
+                _used_paths.add(path)
+
+                try:
+                    executable = _get_executable_path(path, safe=safe)
+                    yield Environment(executable)
+                except InvalidPythonEnvironment:
+                    pass
+
+    return py27_comp(paths, **kwargs)
+
+
+def find_system_environments():
+    """
+    Ignores virtualenvs and returns the Python versions that were installed on
+    your system. This might return nothing, if you're running Python e.g. from
+    a portable version.
+
+    The environments are sorted from latest to oldest Python version.
+
+    :yields: :class:`Environment`
+    """
+    for version_string in _SUPPORTED_PYTHONS:
+        try:
+            yield get_system_environment(version_string)
+        except InvalidPythonEnvironment:
+            pass
+
+
+# TODO: this function should probably return a list of environments since
+# multiple Python installations can be found on a system for the same version.
+def get_system_environment(version):
+    """
+    Return the first Python environment found for a string of the form 'X.Y'
+    where X and Y are the major and minor versions of Python.
+
+    :raises: :exc:`.InvalidPythonEnvironment`
+    :returns: :class:`Environment`
+    """
+    exe = which('python' + version)
+    if exe:
+        if exe == sys.executable:
+            return SameEnvironment()
+        return Environment(exe)
+
+    if os.name == 'nt':
+        for exe in _get_executables_from_windows_registry(version):
+            return Environment(exe)
+    raise InvalidPythonEnvironment("Cannot find executable python%s." % version)
+
+
+def create_environment(path, safe=True):
+    """
+    Make it possible to manually create an Environment object by specifying a
+    Virtualenv path or an executable path.
+
+    :raises: :exc:`.InvalidPythonEnvironment`
+    :returns: :class:`Environment`
+    """
+    if os.path.isfile(path):
+        _assert_safe(path, safe)
+        return Environment(path)
+    return Environment(_get_executable_path(path, safe=safe))
+
+
+def _get_executable_path(path, safe=True):
+    """
+    Returns None if it's not actually a virtual env.
+    """
+
+    if os.name == 'nt':
+        python = os.path.join(path, 'Scripts', 'python.exe')
+    else:
+        python = os.path.join(path, 'bin', 'python')
+    if not os.path.exists(python):
+        raise InvalidPythonEnvironment("%s seems to be missing." % python)
+
+    _assert_safe(python, safe)
+    return python
+
+
+def _get_executables_from_windows_registry(version):
+    # The winreg module is named _winreg on Python 2.
+    try:
+        import winreg
+    except ImportError:
+        import _winreg as winreg
+
+    # TODO: support Python Anaconda.
+    sub_keys = [
+        r'SOFTWARE\Python\PythonCore\{version}\InstallPath',
+        r'SOFTWARE\Wow6432Node\Python\PythonCore\{version}\InstallPath',
+        r'SOFTWARE\Python\PythonCore\{version}-32\InstallPath',
+        r'SOFTWARE\Wow6432Node\Python\PythonCore\{version}-32\InstallPath'
+    ]
+    for root_key in [winreg.HKEY_CURRENT_USER, winreg.HKEY_LOCAL_MACHINE]:
+        for sub_key in sub_keys:
+            sub_key = sub_key.format(version=version)
+            try:
+                with winreg.OpenKey(root_key, sub_key) as key:
+                    prefix = winreg.QueryValueEx(key, '')[0]
+                    exe = os.path.join(prefix, 'python.exe')
+                    if os.path.isfile(exe):
+                        yield exe
+            except WindowsError:
+                pass
+
+
+def _assert_safe(executable_path, safe):
+    if safe and not _is_safe(executable_path):
+        raise InvalidPythonEnvironment(
+            "The python binary is potentially unsafe.")
+
+
+def _is_safe(executable_path):
+    # Resolve sym links. A venv typically is a symlink to a known Python
+    # binary. Only virtualenvs copy symlinks around.
+    real_path = os.path.realpath(executable_path)
+
+    if _is_unix_safe_simple(real_path):
+        return True
+
+    # Just check the list of known Python versions. If it's not in there,
+    # it's likely an attacker or some Python that was not properly
+    # installed in the system.
+    for environment in find_system_environments():
+        if environment.executable == real_path:
+            return True
+
+        # If the versions don't match, just compare the binary files. If we
+        # don't do that, only venvs will be working and not virtualenvs.
+        # venvs are symlinks while virtualenvs are actual copies of the
+        # Python files.
+        # This still means that if the system Python is updated and the
+        # virtualenv's Python is not (which is probably never going to get
+        # upgraded), it will not work with Jedi. IMO that's fine, because
+        # people should just be using venv. ~ dave
+        if environment._sha256 == _calculate_sha256_for_file(real_path):
+            return True
+    return False
+
+
+def _is_unix_safe_simple(real_path):
+    if _is_unix_admin():
+        # In case we are root, just be conservative and
+        # only execute known paths.
+        return any(real_path.startswith(p) for p in _SAFE_PATHS)
+
+    uid = os.stat(real_path).st_uid
+    # The interpreter needs to be owned by root. This means that it wasn't
+    # written by a user and therefore attacking Jedi is not as simple.
+    # The attack could look like the following:
+    # 1. A user clones a repository.
+    # 2. The repository has an innocent looking folder called foobar. jedi
+    #    searches for the folder and executes foobar/bin/python --version if
+    #    there's also a foobar/bin/activate.
+    # 3. The bin/python is obviously not a python script but a bash script or
+    #    whatever the attacker wants.
+    return uid == 0
+
+
+def _is_unix_admin():
+    try:
+        return os.getuid() == 0
+    except AttributeError:
+        return False  # Windows

jedi/api/exceptions.py

@@ -0,0 +1,10 @@
+class _JediError(Exception):
+    pass
+
+
+class InternalError(_JediError):
+    pass
+
+
+class WrongVersion(_JediError):
+    pass

jedi/api/helpers.py

@@ -7,11 +7,12 @@ from textwrap import dedent
 
 from parso.python.parser import Parser
 from parso.python import tree
-from parso import split_lines
 
 from jedi._compatibility import u
+from jedi.evaluate.syntax_tree import eval_atom
 from jedi.evaluate.helpers import evaluate_call_of_leaf
-from jedi.cache import time_cache
+from jedi.evaluate.compiled import get_string_context_set
+from jedi.cache import call_signature_time_cache
 
 
 CompletionParts = namedtuple('CompletionParts', ['path', 'has_dot', 'name'])
@@ -43,7 +44,7 @@ def _get_code(code_lines, start_pos, end_pos):
     lines[-1] = lines[-1][:end_pos[1]]
     # Remove first line indentation.
     lines[0] = lines[0][start_pos[1]:]
-    return '\n'.join(lines)
+    return ''.join(lines)
 
 
 class OnErrorLeaf(Exception):
@@ -52,28 +53,11 @@ class OnErrorLeaf(Exception):
         return self.args[0]
 
 
-def _is_on_comment(leaf, position):
-    comment_lines = split_lines(leaf.prefix)
-    difference = leaf.start_pos[0] - position[0]
-    prefix_start_pos = leaf.get_start_pos_of_prefix()
-    if difference == 0:
-        indent = leaf.start_pos[1]
-    elif position[0] == prefix_start_pos[0]:
-        indent = prefix_start_pos[1]
-    else:
-        indent = 0
-    line = comment_lines[-difference - 1][:position[1] - indent]
-    return '#' in line
-
-
 def _get_code_for_stack(code_lines, module_node, position):
     leaf = module_node.get_leaf_for_position(position, include_prefixes=True)
     # It might happen that we're on whitespace or on a comment. This means
     # that we would not get the right leaf.
     if leaf.start_pos >= position:
-        if _is_on_comment(leaf, position):
-            return u('')
-
         # If we're not on a comment simply get the previous leaf and proceed.
         leaf = leaf.get_previous_leaf()
         if leaf is None:
@@ -121,11 +105,17 @@ def get_stack_at_position(grammar, code_lines, module_node, pos):
         # TODO This is for now not an official parso API that exists purely
         #   for Jedi.
         tokens = grammar._tokenize(code)
-        for token_ in tokens:
-            if token_.string == safeword:
+        for token in tokens:
+            if token.string == safeword:
+                raise EndMarkerReached()
+            elif token.prefix.endswith(safeword):
+                # This happens with comments.
+                raise EndMarkerReached()
+            elif token.string.endswith(safeword):
+                yield token  # Probably an f-string literal that was not finished.
                 raise EndMarkerReached()
             else:
-                yield token_
+                yield token
 
     # The code might be indedented, just remove it.
     code = dedent(_get_code_for_stack(code_lines, module_node, pos))
@@ -133,65 +123,17 @@ def get_stack_at_position(grammar, code_lines, module_node, pos):
     # completion.
     # Use Z as a prefix because it's not part of a number suffix.
     safeword = 'ZZZ_USER_WANTS_TO_COMPLETE_HERE_WITH_JEDI'
-    code = code + safeword
+    code = code + ' ' + safeword
 
     p = Parser(grammar._pgen_grammar, error_recovery=True)
     try:
         p.parse(tokens=tokenize_without_endmarker(code))
     except EndMarkerReached:
-        return Stack(p.pgen_parser.stack)
-    raise SystemError("This really shouldn't happen. There's a bug in Jedi.")
-
-
-class Stack(list):
-    def get_node_names(self, grammar):
-        for dfa, state, (node_number, nodes) in self:
-            yield grammar.number2symbol[node_number]
-
-    def get_nodes(self):
-        for dfa, state, (node_number, nodes) in self:
-            for node in nodes:
-                yield node
-
-
-def get_possible_completion_types(pgen_grammar, stack):
-    def add_results(label_index):
-        try:
-            grammar_labels.append(inversed_tokens[label_index])
-        except KeyError:
-            try:
-                keywords.append(inversed_keywords[label_index])
-            except KeyError:
-                t, v = pgen_grammar.labels[label_index]
-                assert t >= 256
-                # See if it's a symbol and if we're in its first set
-                inversed_keywords
-                itsdfa = pgen_grammar.dfas[t]
-                itsstates, itsfirst = itsdfa
-                for first_label_index in itsfirst.keys():
-                    add_results(first_label_index)
-
-    inversed_keywords = dict((v, k) for k, v in pgen_grammar.keywords.items())
-    inversed_tokens = dict((v, k) for k, v in pgen_grammar.tokens.items())
-
-    keywords = []
-    grammar_labels = []
-
-    def scan_stack(index):
-        dfa, state, node = stack[index]
-        states, first = dfa
-        arcs = states[state]
-
-        for label_index, new_state in arcs:
-            if label_index == 0:
-                # An accepting state, check the stack below.
-                scan_stack(index - 1)
-            else:
-                add_results(label_index)
-
-    scan_stack(-1)
-
-    return keywords, grammar_labels
+        return p.stack
+    raise SystemError(
+        "This really shouldn't happen. There's a bug in Jedi:\n%s"
+        % list(tokenize_without_endmarker(code))
+    )
 
 
 def evaluate_goto_definition(evaluator, context, leaf):
@@ -206,7 +148,9 @@ def evaluate_goto_definition(evaluator, context, leaf):
     elif parent.type == 'trailer':
         return evaluate_call_of_leaf(context, leaf)
     elif isinstance(leaf, tree.Literal):
-        return context.evaluator.eval_atom(context, leaf)
+        return eval_atom(context, leaf)
+    elif leaf.type in ('fstring_string', 'fstring_start', 'fstring_end'):
+        return get_string_context_set(evaluator)
     return []
 
 
@@ -228,7 +172,8 @@ def _get_index_and_key(nodes, position):
 
     if nodes_before:
         last = nodes_before[-1]
-        if last.type == 'argument' and last.children[1].end_pos <= position:
+        if last.type == 'argument' and last.children[1] == '=' \
+                and last.children[1].end_pos <= position:
             # Checked if the argument
             key_str = last.children[0].value
         elif last == '=':
@@ -293,14 +238,14 @@ def get_call_signature_details(module, position):
     return None
 
 
-@time_cache("call_signatures_validity")
+@call_signature_time_cache("call_signatures_validity")
 def cache_call_signatures(evaluator, context, bracket_leaf, code_lines, user_pos):
     """This function calculates the cache key."""
-    index = user_pos[0] - 1
+    line_index = user_pos[0] - 1
 
-    before_cursor = code_lines[index][:user_pos[1]]
-    other_lines = code_lines[bracket_leaf.start_pos[0]:index]
-    whole = '\n'.join(other_lines + [before_cursor])
+    before_cursor = code_lines[line_index][:user_pos[1]]
+    other_lines = code_lines[bracket_leaf.start_pos[0]:line_index]
+    whole = ''.join(other_lines + [before_cursor])
     before_bracket = re.match(r'.*\(', whole, re.DOTALL)
 
     module_path = context.get_root_context().py__file__()

jedi/api/interpreter.py

@@ -2,10 +2,17 @@
 TODO Some parts of this module are still not well documented.
 """
 
-from jedi.evaluate.representation import ModuleContext
+from jedi.evaluate.context import ModuleContext
 from jedi.evaluate import compiled
 from jedi.evaluate.compiled import mixed
-from jedi.evaluate.context import Context
+from jedi.evaluate.compiled.access import create_access_path
+from jedi.evaluate.base_context import Context
+
+
+def _create(evaluator, obj):
+    return compiled.create_from_access_path(
+        evaluator, create_access_path(evaluator, obj)
+    )
 
 
 class NamespaceObject(object):
@@ -14,15 +21,18 @@ class NamespaceObject(object):
 
 
 class MixedModuleContext(Context):
-    resets_positions = True
     type = 'mixed_module'
 
-    def __init__(self, evaluator, tree_module, namespaces, path):
+    def __init__(self, evaluator, tree_module, namespaces, path, code_lines):
         self.evaluator = evaluator
         self._namespaces = namespaces
 
         self._namespace_objects = [NamespaceObject(n) for n in namespaces]
-        self._module_context = ModuleContext(evaluator, tree_module, path=path)
+        self._module_context = ModuleContext(
+            evaluator, tree_module,
+            path=path,
+            code_lines=code_lines
+        )
         self.tree_node = tree_module
 
     def get_node(self):
@@ -33,7 +43,7 @@ class MixedModuleContext(Context):
             yield filter
 
         for namespace_obj in self._namespace_objects:
-            compiled_object = compiled.create(self.evaluator, namespace_obj)
+            compiled_object = _create(self.evaluator, namespace_obj)
             mixed_object = mixed.MixedObject(
                 self.evaluator,
                 parent_context=self,
@@ -43,5 +53,9 @@ class MixedModuleContext(Context):
             for filter in mixed_object.get_filters(*args, **kwargs):
                 yield filter
 
+    @property
+    def code_lines(self):
+        return self._module_context.code_lines
+
     def __getattr__(self, name):
         return getattr(self._module_context, name)

jedi/api/keywords.py

@@ -1,10 +1,7 @@
 import pydoc
-import keyword
 
-from jedi._compatibility import is_py3, is_py35
-from jedi import common
+from jedi.evaluate.utils import ignored
 from jedi.evaluate.filters import AbstractNameDefinition
-from parso.python.tree import Leaf
 
 try:
     from pydoc_data import topics as pydoc_topics
@@ -17,87 +14,30 @@ except ImportError:
         # pydoc_data module in its file python3x.zip.
         pydoc_topics = None
 
-if is_py3:
-    if is_py35:
-        # in python 3.5 async and await are not proper keywords, but for
-        # completion pursposes should as as though they are
-        keys = keyword.kwlist + ["async", "await"]
-    else:
-        keys = keyword.kwlist
-else:
-    keys = keyword.kwlist + ['None', 'False', 'True']
-
-
-def has_inappropriate_leaf_keyword(pos, module):
-    relevant_errors = filter(
-        lambda error: error.first_pos[0] == pos[0],
-        module.error_statement_stacks)
-
-    for error in relevant_errors:
-        if error.next_token in keys:
-            return True
-
-    return False
-
-
-def completion_names(evaluator, stmt, pos, module):
-    keyword_list = all_keywords(evaluator)
-
-    if not isinstance(stmt, Leaf) or has_inappropriate_leaf_keyword(pos, module):
-        keyword_list = filter(
-            lambda keyword: not keyword.only_valid_as_leaf,
-            keyword_list
-        )
-    return [keyword.name for keyword in keyword_list]
-
-
-def all_keywords(evaluator, pos=(0, 0)):
-    return set([Keyword(evaluator, k, pos) for k in keys])
-
-
-def keyword(evaluator, string, pos=(0, 0)):
-    if string in keys:
-        return Keyword(evaluator, string, pos)
-    else:
-        return None
-
 
 def get_operator(evaluator, string, pos):
     return Keyword(evaluator, string, pos)
 
 
-keywords_only_valid_as_leaf = (
-    'continue',
-    'break',
-)
-
-
 class KeywordName(AbstractNameDefinition):
-    api_type = 'keyword'
+    api_type = u'keyword'
 
     def __init__(self, evaluator, name):
         self.evaluator = evaluator
         self.string_name = name
-        self.parent_context = evaluator.BUILTINS
-
-    def eval(self):
-        return set()
+        self.parent_context = evaluator.builtins_module
 
     def infer(self):
         return [Keyword(self.evaluator, self.string_name, (0, 0))]
 
 
 class Keyword(object):
-    api_type = 'keyword'
+    api_type = u'keyword'
 
     def __init__(self, evaluator, name, pos):
         self.name = KeywordName(evaluator, name)
         self.start_pos = pos
-        self.parent = evaluator.BUILTINS
-
-    @property
-    def only_valid_as_leaf(self):
-        return self.name.value in keywords_only_valid_as_leaf
+        self.parent = evaluator.builtins_module
 
     @property
     def names(self):
@@ -123,7 +63,7 @@ def imitate_pydoc(string):
     # with unicode strings)
     string = str(string)
     h = pydoc.help
-    with common.ignored(KeyError):
+    with ignored(KeyError):
         # try to access symbols
         string = h.symbols[string]
         string, _, related = string.partition(' ')

jedi/api/project.py

@@ -0,0 +1,195 @@
+import os
+import json
+
+from jedi._compatibility import FileNotFoundError, NotADirectoryError, PermissionError
+from jedi.api.environment import SameEnvironment, \
+    get_cached_default_environment
+from jedi.api.exceptions import WrongVersion
+from jedi._compatibility import force_unicode
+from jedi.evaluate.sys_path import discover_buildout_paths
+from jedi.evaluate.cache import evaluator_as_method_param_cache
+from jedi.common.utils import traverse_parents
+
+_CONFIG_FOLDER = '.jedi'
+_CONTAINS_POTENTIAL_PROJECT = 'setup.py', '.git', '.hg', 'requirements.txt', 'MANIFEST.in'
+
+_SERIALIZER_VERSION = 1
+
+
+def _remove_duplicates_from_path(path):
+    used = set()
+    for p in path:
+        if p in used:
+            continue
+        used.add(p)
+        yield p
+
+
+def _force_unicode_list(lst):
+    return list(map(force_unicode, lst))
+
+
+class Project(object):
+    # TODO serialize environment
+    _serializer_ignore_attributes = ('_environment',)
+    _environment = None
+
+    @staticmethod
+    def _get_json_path(base_path):
+        return os.path.join(base_path, _CONFIG_FOLDER, 'project.json')
+
+    @classmethod
+    def load(cls, path):
+        """
+        :param path: The path of the directory you want to use as a project.
+        """
+        with open(cls._get_json_path(path)) as f:
+            version, data = json.load(f)
+
+        if version == 1:
+            self = cls.__new__()
+            self.__dict__.update(data)
+            return self
+        else:
+            raise WrongVersion(
+                "The Jedi version of this project seems newer than what we can handle."
+            )
+
+    def __init__(self, path, **kwargs):
+        """
+        :param path: The base path for this project.
+        :param sys_path: list of str. You can override the sys path if you
+            want. By default the ``sys.path.`` is generated from the
+            environment (virtualenvs, etc).
+        :param smart_sys_path: If this is enabled (default), adds paths from
+            local directories. Otherwise you will have to rely on your packages
+            being properly configured on the ``sys.path``.
+        """
+        def py2_comp(path, environment=None, sys_path=None,
+                     smart_sys_path=True, _django=False):
+            self._path = path
+            if isinstance(environment, SameEnvironment):
+                self._environment = environment
+
+            self._sys_path = sys_path
+            self._smart_sys_path = smart_sys_path
+            self._django = _django
+
+        py2_comp(path, **kwargs)
+
+    def _get_base_sys_path(self, environment=None):
+        if self._sys_path is not None:
+            return self._sys_path
+
+        # The sys path has not been set explicitly.
+        if environment is None:
+            environment = self.get_environment()
+
+        sys_path = list(environment.get_sys_path())
+        try:
+            sys_path.remove('')
+        except ValueError:
+            pass
+        return sys_path
+
+    @evaluator_as_method_param_cache()
+    def _get_sys_path(self, evaluator, environment=None):
+        """
+        Keep this method private for all users of jedi. However internally this
+        one is used like a public method.
+        """
+        suffixed = []
+        prefixed = []
+
+        sys_path = list(self._get_base_sys_path(environment))
+        if self._smart_sys_path:
+            prefixed.append(self._path)
+
+            if evaluator.script_path is not None:
+                suffixed += discover_buildout_paths(evaluator, evaluator.script_path)
+
+                traversed = list(traverse_parents(evaluator.script_path))
+
+                # AFAIK some libraries have imports like `foo.foo.bar`, which
+                # leads to the conclusion to by default prefer longer paths
+                # rather than shorter ones by default.
+                suffixed += reversed(traversed)
+
+        if self._django:
+            prefixed.append(self._path)
+
+        path = prefixed + sys_path + suffixed
+        return list(_force_unicode_list(_remove_duplicates_from_path(path)))
+
+    def save(self):
+        data = dict(self.__dict__)
+        for attribute in self._serializer_ignore_attributes:
+            data.pop(attribute, None)
+
+        with open(self._get_json_path(self._path), 'wb') as f:
+            return json.dump((_SERIALIZER_VERSION, data), f)
+
+    def get_environment(self):
+        if self._environment is None:
+            return get_cached_default_environment()
+
+        return self._environment
+
+    def __repr__(self):
+        return '<%s: %s>' % (self.__class__.__name__, self._path)
+
+
+def _is_potential_project(path):
+    for name in _CONTAINS_POTENTIAL_PROJECT:
+        if os.path.exists(os.path.join(path, name)):
+            return True
+    return False
+
+
+def _is_django_path(directory):
+    """ Detects the path of the very well known Django library (if used) """
+    try:
+        with open(os.path.join(directory, 'manage.py'), 'rb') as f:
+            return b"DJANGO_SETTINGS_MODULE" in f.read()
+    except (FileNotFoundError, NotADirectoryError, PermissionError):
+        return False
+
+    return False
+
+
+def get_default_project(path=None):
+    if path is None:
+        path = os.getcwd()
+
+    check = os.path.realpath(path)
+    probable_path = None
+    first_no_init_file = None
+    for dir in traverse_parents(check, include_current=True):
+        try:
+            return Project.load(dir)
+        except (FileNotFoundError, NotADirectoryError, PermissionError):
+            pass
+
+        if first_no_init_file is None:
+            if os.path.exists(os.path.join(dir, '__init__.py')):
+                # In the case that a __init__.py exists, it's in 99% just a
+                # Python package and the project sits at least one level above.
+                continue
+            else:
+                first_no_init_file = dir
+
+        if _is_django_path(dir):
+            return Project(dir, _django=True)
+
+        if probable_path is None and _is_potential_project(dir):
+            probable_path = dir
+
+    if probable_path is not None:
+        # TODO search for setup.py etc
+        return Project(probable_path)
+
+    if first_no_init_file is not None:
+        return Project(first_no_init_file)
+
+    curdir = path if os.path.isdir(path) else os.path.dirname(path)
+    return Project(curdir)

jedi/api/replstartup.py

@@ -1,6 +1,8 @@
 """
 To use Jedi completion in Python interpreter, add the following in your shell
-setup (e.g., ``.bashrc``)::
+setup (e.g., ``.bashrc``). This works only on Linux/Mac, because readline is
+not available on Windows. If you still want Jedi autocompletion in your REPL,
+just use IPython instead::
 
     export PYTHONSTARTUP="$(python -m jedi repl)"
 
@@ -11,8 +13,8 @@ Then you will be able to use Jedi completer in your Python interpreter::
     [GCC 4.6.1] on linux2
     Type "help", "copyright", "credits" or "license" for more information.
     >>> import os
-    >>> os.path.join().split().in<TAB>                     # doctest: +SKIP
-    os.path.join().split().index   os.path.join().split().insert
+    >>> os.path.join('a', 'b').split().in<TAB>            # doctest: +SKIP
+    ..dex   ..sert
 
 """
 import jedi.utils

jedi/api/usages.py

@@ -1,75 +0,0 @@
-from jedi.api import classes
-from parso.python import tree
-from jedi.evaluate import imports
-from jedi.evaluate.filters import TreeNameDefinition
-from jedi.evaluate.representation import ModuleContext
-
-
-def compare_contexts(c1, c2):
-    return c1 == c2 or (c1[1] == c2[1] and c1[0].tree_node == c2[0].tree_node)
-
-
-def usages(evaluator, definition_names, mods):
-    """
-    :param definitions: list of Name
-    """
-    def resolve_names(definition_names):
-        for name in definition_names:
-            if name.api_type == 'module':
-                found = False
-                for context in name.infer():
-                    if isinstance(context, ModuleContext):
-                        found = True
-                        yield context.name
-                if not found:
-                    yield name
-            else:
-                yield name
-
-    def compare_array(definition_names):
-        """ `definitions` are being compared by module/start_pos, because
-        sometimes the id's of the objects change (e.g. executions).
-        """
-        return [
-            (name.get_root_context(), name.start_pos)
-            for name in resolve_names(definition_names)
-        ]
-
-    search_name = list(definition_names)[0].string_name
-    compare_definitions = compare_array(definition_names)
-    mods = mods | set([d.get_root_context() for d in definition_names])
-    definition_names = set(resolve_names(definition_names))
-    for m in imports.get_modules_containing_name(evaluator, mods, search_name):
-        if isinstance(m, ModuleContext):
-            for name_node in m.tree_node.get_used_names().get(search_name, []):
-                context = evaluator.create_context(m, name_node)
-                result = evaluator.goto(context, name_node)
-                if any(compare_contexts(c1, c2)
-                       for c1 in compare_array(result)
-                       for c2 in compare_definitions):
-                    name = TreeNameDefinition(context, name_node)
-                    definition_names.add(name)
-                    # Previous definitions might be imports, so include them
-                    # (because goto might return that import name).
-                    compare_definitions += compare_array([name])
-        else:
-            # compiled objects
-            definition_names.add(m.name)
-
-    return [classes.Definition(evaluator, n) for n in definition_names]
-
-
-def resolve_potential_imports(evaluator, definitions):
-    """ Adds the modules of the imports """
-    new = set()
-    for d in definitions:
-        if isinstance(d, TreeNameDefinition):
-            imp_or_stmt = d.tree_name.get_definition()
-            if isinstance(imp_or_stmt, tree.Import):
-                new |= resolve_potential_imports(
-                    evaluator,
-                    set(imports.infer_import(
-                        d.parent_context, d.tree_name, is_goto=True
-                    ))
-                )
-    return set(definitions) | new

jedi/cache.py

@@ -12,7 +12,7 @@ there are global variables, which are holding the cache information. Some of
 these variables are being cleaned after every API usage.
 """
 import time
-import inspect
+from functools import wraps
 
 from jedi import settings
 from parso.cache import parser_cache
@@ -46,8 +46,6 @@ def underscore_memoization(func):
             return getattr(self, name)
         except AttributeError:
             result = func(self)
-            if inspect.isgenerator(result):
-                result = list(result)
             setattr(self, name, result)
             return result
 
@@ -77,7 +75,7 @@ def clear_time_caches(delete_all=False):
                     del tc[key]
 
 
-def time_cache(time_add_setting):
+def call_signature_time_cache(time_add_setting):
     """
     This decorator works as follows: Call it with a setting and after that
     use the function with a callable that returns the key.
@@ -109,8 +107,32 @@ def time_cache(time_add_setting):
     return _temp
 
 
+def time_cache(seconds):
+    def decorator(func):
+        cache = {}
+
+        @wraps(func)
+        def wrapper(*args, **kwargs):
+            key = (args, frozenset(kwargs.items()))
+            try:
+                created, result = cache[key]
+                if time.time() < created + seconds:
+                    return result
+            except KeyError:
+                pass
+            result = func(*args, **kwargs)
+            cache[key] = time.time(), result
+            return result
+
+        wrapper.clear_cache = lambda: cache.clear()
+        return wrapper
+
+    return decorator
+
+
 def memoize_method(method):
     """A normal memoize function."""
+    @wraps(method)
     def wrapper(self, *args, **kwargs):
         cache_dict = self.__dict__.setdefault('_memoize_method_dct', {})
         dct = cache_dict.setdefault(method, {})

jedi/common/__init__.py

@@ -0,0 +1 @@
+from jedi.common.context import BaseContextSet, BaseContext

jedi/common/context.py

@@ -0,0 +1,67 @@
+class BaseContext(object):
+    def __init__(self, evaluator, parent_context=None):
+        self.evaluator = evaluator
+        self.parent_context = parent_context
+
+    def get_root_context(self):
+        context = self
+        while True:
+            if context.parent_context is None:
+                return context
+            context = context.parent_context
+
+
+class BaseContextSet(object):
+    def __init__(self, *args):
+        self._set = set(args)
+
+    @classmethod
+    def from_iterable(cls, iterable):
+        return cls.from_set(set(iterable))
+
+    @classmethod
+    def from_set(cls, set_):
+        self = cls()
+        self._set = set_
+        return self
+
+    @classmethod
+    def from_sets(cls, sets):
+        """
+        Used to work with an iterable of set.
+        """
+        aggregated = set()
+        sets = list(sets)
+        for set_ in sets:
+            if isinstance(set_, BaseContextSet):
+                aggregated |= set_._set
+            else:
+                aggregated |= set_
+        return cls.from_set(aggregated)
+
+    def __or__(self, other):
+        return type(self).from_set(self._set | other._set)
+
+    def __iter__(self):
+        for element in self._set:
+            yield element
+
+    def __bool__(self):
+        return bool(self._set)
+
+    def __len__(self):
+        return len(self._set)
+
+    def __repr__(self):
+        return '%s(%s)' % (self.__class__.__name__, ', '.join(str(s) for s in self._set))
+
+    def filter(self, filter_func):
+        return type(self).from_iterable(filter(filter_func, self._set))
+
+    def __getattr__(self, name):
+        def mapper(*args, **kwargs):
+            return type(self).from_sets(
+                getattr(context, name)(*args, **kwargs)
+                for context in self._set
+            )
+        return mapper

jedi/common/utils.py

@@ -0,0 +1,26 @@
+import os
+from contextlib import contextmanager
+
+
+def traverse_parents(path, include_current=False):
+    if not include_current:
+        path = os.path.dirname(path)
+
+    previous = None
+    while previous != path:
+        yield path
+        previous = path
+        path = os.path.dirname(path)
+
+
+@contextmanager
+def monkeypatch(obj, attribute_name, new_value):
+    """
+    Like pytest's monkeypatch, but as a context manager.
+    """
+    old_value = getattr(obj, attribute_name)
+    try:
+        setattr(obj, attribute_name, new_value)
+        yield
+    finally:
+        setattr(obj, attribute_name, old_value)

jedi/debug.py

@@ -2,16 +2,17 @@ from jedi._compatibility import encoding, is_py3, u
 import os
 import time
 
+_inited = False
+
+
 def _lazy_colorama_init():
     """
-    Lazily init colorama if necessary, not to screw up stdout is debug not
-    enabled.
+    Lazily init colorama if necessary, not to screw up stdout if debugging is
+    not enabled.
 
     This version of the function does nothing.
     """
-    pass
 
-_inited=False
 
 try:
     if os.name == 'nt':
@@ -21,7 +22,8 @@ try:
         # Use colorama for nicer console output.
         from colorama import Fore, init
         from colorama import initialise
-        def _lazy_colorama_init():
+
+        def _lazy_colorama_init():  # noqa: F811
             """
             Lazily init colorama if necessary, not to screw up stdout is
             debug not enabled.
@@ -35,7 +37,7 @@ try:
                 # need this.
                 initialise.atexit_done = True
                 try:
-                    init()
+                    init(strip=False)
                 except Exception:
                     # Colorama fails with initializing under vim and is buggy in
                     # version 0.3.6.

jedi/evaluate/__init__.py

@@ -12,29 +12,32 @@ Evaluation of Python code in |jedi| is based on three assumptions:
 * The programmer is not a total dick, e.g. like `this
   <https://github.com/davidhalter/jedi/issues/24>`_ :-)
 
-The actual algorithm is based on a principle called lazy evaluation. If you
-don't know about it, google it.  That said, the typical entry point for static
-analysis is calling ``eval_statement``. There's separate logic for
-autocompletion in the API, the evaluator is all about evaluating an expression.
+The actual algorithm is based on a principle called lazy evaluation.  That
+said, the typical entry point for static analysis is calling
+``eval_expr_stmt``. There's separate logic for autocompletion in the API, the
+evaluator is all about evaluating an expression.
 
-Now you need to understand what follows after ``eval_statement``. Let's
+TODO this paragraph is not what jedi does anymore, it's similar, but not the
+same.
+
+Now you need to understand what follows after ``eval_expr_stmt``. Let's
 make an example::
 
     import datetime
     datetime.date.toda# <-- cursor here
 
 First of all, this module doesn't care about completion. It really just cares
-about ``datetime.date``. At the end of the procedure ``eval_statement`` will
+about ``datetime.date``. At the end of the procedure ``eval_expr_stmt`` will
 return the ``date`` class.
 
 To *visualize* this (simplified):
 
-- ``Evaluator.eval_statement`` doesn't do much, because there's no assignment.
-- ``Evaluator.eval_element`` cares for resolving the dotted path
+- ``Evaluator.eval_expr_stmt`` doesn't do much, because there's no assignment.
+- ``Context.eval_node`` cares for resolving the dotted path
 - ``Evaluator.find_types`` searches for global definitions of datetime, which
   it finds in the definition of an import, by scanning the syntax tree.
 - Using the import logic, the datetime module is found.
-- Now ``find_types`` is called again by ``eval_element`` to find ``date``
+- Now ``find_types`` is called again by ``eval_node`` to find ``date``
   inside the datetime module.
 
 Now what would happen if we wanted ``datetime.date.foo.bar``? Two more
@@ -46,7 +49,7 @@ What if the import would contain another ``ExprStmt`` like this::
     from foo import bar
     Date = bar.baz
 
-Well... You get it. Just another ``eval_statement`` recursion. It's really
+Well... You get it. Just another ``eval_expr_stmt`` recursion. It's really
 easy. Python can obviously get way more complicated then this. To understand
 tuple assignments, list comprehensions and everything else, a lot more code had
 to be written.
@@ -60,163 +63,74 @@ only *evaluates* what needs to be *evaluated*. All the statements and modules
 that are not used are just being ignored.
 """
 
-import copy
-import sys
-
 from parso.python import tree
 import parso
+from parso import python_bytes_to_unicode
 
 from jedi import debug
-from jedi.common import unite
-from jedi.evaluate import representation as er
+from jedi import parser_utils
+from jedi.evaluate.utils import unite
 from jedi.evaluate import imports
 from jedi.evaluate import recursion
-from jedi.evaluate import iterable
 from jedi.evaluate.cache import evaluator_function_cache
-from jedi.evaluate import stdlib
-from jedi.evaluate import finder
 from jedi.evaluate import compiled
-from jedi.evaluate import precedence
-from jedi.evaluate import param
 from jedi.evaluate import helpers
-from jedi.evaluate import pep0484
 from jedi.evaluate.filters import TreeNameDefinition, ParamName
-from jedi.evaluate.instance import AnonymousInstance, BoundMethod
-from jedi.evaluate.context import ContextualizedName, ContextualizedNode
-from jedi import parser_utils
-
-
-def _limit_context_infers(func):
-    """
-    This is for now the way how we limit type inference going wild. There are
-    other ways to ensure recursion limits as well. This is mostly necessary
-    because of instance (self) access that can be quite tricky to limit.
-
-    I'm still not sure this is the way to go, but it looks okay for now and we
-    can still go anther way in the future. Tests are there. ~ dave
-    """
-    def wrapper(evaluator, context, *args, **kwargs):
-        n = context.tree_node
-        try:
-            evaluator.inferred_element_counts[n] += 1
-            if evaluator.inferred_element_counts[n] > 300:
-                debug.warning('In context %s there were too many inferences.', n)
-                return set()
-        except KeyError:
-            evaluator.inferred_element_counts[n] = 1
-        return func(evaluator, context, *args, **kwargs)
-
-    return wrapper
+from jedi.evaluate.base_context import ContextualizedName, ContextualizedNode, \
+    ContextSet, NO_CONTEXTS, iterate_contexts
+from jedi.evaluate.context import ClassContext, FunctionContext, \
+    AnonymousInstance, BoundMethod
+from jedi.evaluate.context.iterable import CompForContext
+from jedi.evaluate.syntax_tree import eval_trailer, eval_expr_stmt, \
+    eval_node, check_tuple_assignments
 
 
 class Evaluator(object):
-    def __init__(self, grammar, sys_path=None):
-        self.grammar = grammar
+    def __init__(self, project, environment=None, script_path=None):
+        if environment is None:
+            environment = project.get_environment()
+        self.environment = environment
+        self.script_path = script_path
+        self.compiled_subprocess = environment.get_evaluator_subprocess(self)
+        self.grammar = environment.get_grammar()
+
         self.latest_grammar = parso.load_grammar(version='3.6')
         self.memoize_cache = {}  # for memoize decorators
-        # To memorize modules -> equals `sys.modules`.
-        self.modules = {}  # like `sys.modules`.
+        self.module_cache = imports.ModuleCache()  # does the job of `sys.modules`.
         self.compiled_cache = {}  # see `evaluate.compiled.create()`
         self.inferred_element_counts = {}
         self.mixed_cache = {}  # see `evaluate.compiled.mixed._create()`
         self.analysis = []
         self.dynamic_params_depth = 0
         self.is_analysis = False
-        self.python_version = sys.version_info[:2]
-
-        if sys_path is None:
-            sys_path = sys.path
-        self.sys_path = copy.copy(sys_path)
-        try:
-            self.sys_path.remove('')
-        except ValueError:
-            pass
+        self.project = project
+        self.access_cache = {}
+        # This setting is only temporary to limit the work we have to do with
+        # tensorflow and others.
+        self.infer_enabled = True
 
         self.reset_recursion_limitations()
+        self.allow_different_encoding = True
 
-        # Constants
-        self.BUILTINS = compiled.get_special_object(self, 'BUILTINS')
+    @property
+    @evaluator_function_cache()
+    def builtins_module(self):
+        return compiled.get_special_object(self, u'BUILTINS')
 
     def reset_recursion_limitations(self):
         self.recursion_detector = recursion.RecursionDetector()
         self.execution_recursion_detector = recursion.ExecutionRecursionDetector(self)
 
-    def find_types(self, context, name_or_str, name_context, position=None,
-                   search_global=False, is_goto=False, analysis_errors=True):
-        """
-        This is the search function. The most important part to debug.
-        `remove_statements` and `filter_statements` really are the core part of
-        this completion.
-
-        :param position: Position of the last statement -> tuple of line, column
-        :return: List of Names. Their parents are the types.
-        """
-        f = finder.NameFinder(self, context, name_context, name_or_str,
-                              position, analysis_errors=analysis_errors)
-        filters = f.get_filters(search_global)
-        if is_goto:
-            return f.filter_name(filters)
-        return f.find(filters, attribute_lookup=not search_global)
-
-    @_limit_context_infers
-    def eval_statement(self, context, stmt, seek_name=None):
-        with recursion.execution_allowed(self, stmt) as allowed:
-            if allowed or context.get_root_context() == self.BUILTINS:
-                return self._eval_stmt(context, stmt, seek_name)
-        return set()
-
-    #@evaluator_function_cache(default=[])
-    @debug.increase_indent
-    def _eval_stmt(self, context, stmt, seek_name=None):
-        """
-        The starting point of the completion. A statement always owns a call
-        list, which are the calls, that a statement does. In case multiple
-        names are defined in the statement, `seek_name` returns the result for
-        this name.
-
-        :param stmt: A `tree.ExprStmt`.
-        """
-        debug.dbg('eval_statement %s (%s)', stmt, seek_name)
-        rhs = stmt.get_rhs()
-        types = self.eval_element(context, rhs)
-
-        if seek_name:
-            c_node = ContextualizedName(context, seek_name)
-            types = finder.check_tuple_assignments(self, c_node, types)
-
-        first_operator = next(stmt.yield_operators(), None)
-        if first_operator not in ('=', None) and first_operator.type == 'operator':
-            # `=` is always the last character in aug assignments -> -1
-            operator = copy.copy(first_operator)
-            operator.value = operator.value[:-1]
-            name = stmt.get_defined_names()[0].value
-            left = context.py__getattribute__(
-                name, position=stmt.start_pos, search_global=True)
-
-            for_stmt = tree.search_ancestor(stmt, 'for_stmt')
-            if for_stmt is not None and for_stmt.type == 'for_stmt' and types \
-                    and parser_utils.for_stmt_defines_one_name(for_stmt):
-                # Iterate through result and add the values, that's possible
-                # only in for loops without clutter, because they are
-                # predictable. Also only do it, if the variable is not a tuple.
-                node = for_stmt.get_testlist()
-                cn = ContextualizedNode(context, node)
-                ordered = list(iterable.py__iter__(self, cn.infer(), cn))
-
-                for lazy_context in ordered:
-                    dct = {for_stmt.children[1].value: lazy_context.infer()}
-                    with helpers.predefine_names(context, for_stmt, dct):
-                        t = self.eval_element(context, rhs)
-                        left = precedence.calculate(self, context, left, operator, t)
-                types = left
-            else:
-                types = precedence.calculate(self, context, left, operator, types)
-        debug.dbg('eval_statement result %s', types)
-        return types
+    def get_sys_path(self):
+        """Convenience function"""
+        return self.project._get_sys_path(self, environment=self.environment)
 
     def eval_element(self, context, element):
-        if isinstance(context, iterable.CompForContext):
-            return self._eval_element_not_cached(context, element)
+        if not self.infer_enabled:
+            return NO_CONTEXTS
+
+        if isinstance(context, CompForContext):
+            return eval_node(context, element)
 
         if_stmt = element
         while if_stmt is not None:
@@ -227,7 +141,11 @@ class Evaluator(object):
                 if_stmt = None
                 break
         predefined_if_name_dict = context.predefined_names.get(if_stmt)
-        if predefined_if_name_dict is None and if_stmt and if_stmt.type == 'if_stmt':
+        # TODO there's a lot of issues with this one. We actually should do
+        # this in a different way. Caching should only be active in certain
+        # cases and this all sucks.
+        if predefined_if_name_dict is None and if_stmt \
+                and if_stmt.type == 'if_stmt' and self.is_analysis:
             if_stmt_test = if_stmt.children[1]
             name_dicts = [{}]
             # If we already did a check, we don't want to do it again -> If
@@ -261,23 +179,23 @@ class Evaluator(object):
                                 new_name_dicts = list(original_name_dicts)
                                 for i, name_dict in enumerate(new_name_dicts):
                                     new_name_dicts[i] = name_dict.copy()
-                                    new_name_dicts[i][if_name.value] = set([definition])
+                                    new_name_dicts[i][if_name.value] = ContextSet(definition)
 
                                 name_dicts += new_name_dicts
                         else:
                             for name_dict in name_dicts:
                                 name_dict[if_name.value] = definitions
             if len(name_dicts) > 1:
-                result = set()
+                result = ContextSet()
                 for name_dict in name_dicts:
                     with helpers.predefine_names(context, if_stmt, name_dict):
-                        result |= self._eval_element_not_cached(context, element)
+                        result |= eval_node(context, element)
                 return result
             else:
                 return self._eval_element_if_evaluated(context, element)
         else:
             if predefined_if_name_dict:
-                return self._eval_element_not_cached(context, element)
+                return eval_node(context, element)
             else:
                 return self._eval_element_if_evaluated(context, element)
 
@@ -290,201 +208,32 @@ class Evaluator(object):
             parent = parent.parent
             predefined_if_name_dict = context.predefined_names.get(parent)
             if predefined_if_name_dict is not None:
-                return self._eval_element_not_cached(context, element)
+                return eval_node(context, element)
         return self._eval_element_cached(context, element)
 
-    @evaluator_function_cache(default=set())
+    @evaluator_function_cache(default=NO_CONTEXTS)
     def _eval_element_cached(self, context, element):
-        return self._eval_element_not_cached(context, element)
-
-    @debug.increase_indent
-    @_limit_context_infers
-    def _eval_element_not_cached(self, context, element):
-        debug.dbg('eval_element %s@%s', element, element.start_pos)
-        types = set()
-        typ = element.type
-        if typ in ('name', 'number', 'string', 'atom'):
-            types = self.eval_atom(context, element)
-        elif typ == 'keyword':
-            # For False/True/None
-            if element.value in ('False', 'True', 'None'):
-                types.add(compiled.builtin_from_name(self, element.value))
-            # else: print e.g. could be evaluated like this in Python 2.7
-        elif typ == 'lambdef':
-            types = set([er.FunctionContext(self, context, element)])
-        elif typ == 'expr_stmt':
-            types = self.eval_statement(context, element)
-        elif typ in ('power', 'atom_expr'):
-            first_child = element.children[0]
-            if not (first_child.type == 'keyword' and first_child.value == 'await'):
-                types = self.eval_atom(context, first_child)
-                for trailer in element.children[1:]:
-                    if trailer == '**':  # has a power operation.
-                        right = self.eval_element(context, element.children[2])
-                        types = set(precedence.calculate(self, context, types, trailer, right))
-                        break
-                    types = self.eval_trailer(context, types, trailer)
-        elif typ in ('testlist_star_expr', 'testlist',):
-            # The implicit tuple in statements.
-            types = set([iterable.SequenceLiteralContext(self, context, element)])
-        elif typ in ('not_test', 'factor'):
-            types = self.eval_element(context, element.children[-1])
-            for operator in element.children[:-1]:
-                types = set(precedence.factor_calculate(self, types, operator))
-        elif typ == 'test':
-            # `x if foo else y` case.
-            types = (self.eval_element(context, element.children[0]) |
-                     self.eval_element(context, element.children[-1]))
-        elif typ == 'operator':
-            # Must be an ellipsis, other operators are not evaluated.
-            # In Python 2 ellipsis is coded as three single dot tokens, not
-            # as one token 3 dot token.
-            assert element.value in ('.', '...')
-            types = set([compiled.create(self, Ellipsis)])
-        elif typ == 'dotted_name':
-            types = self.eval_atom(context, element.children[0])
-            for next_name in element.children[2::2]:
-                # TODO add search_global=True?
-                types = unite(
-                    typ.py__getattribute__(next_name, name_context=context)
-                    for typ in types
-                )
-            types = types
-        elif typ == 'eval_input':
-            types = self._eval_element_not_cached(context, element.children[0])
-        elif typ == 'annassign':
-            types = pep0484._evaluate_for_annotation(context, element.children[1])
-        else:
-            types = precedence.calculate_children(self, context, element.children)
-        debug.dbg('eval_element result %s', types)
-        return types
-
-    def eval_atom(self, context, atom):
-        """
-        Basically to process ``atom`` nodes. The parser sometimes doesn't
-        generate the node (because it has just one child). In that case an atom
-        might be a name or a literal as well.
-        """
-        if atom.type == 'name':
-            # This is the first global lookup.
-            stmt = tree.search_ancestor(
-                atom, 'expr_stmt', 'lambdef'
-            ) or atom
-            if stmt.type == 'lambdef':
-                stmt = atom
-            return context.py__getattribute__(
-                name_or_str=atom,
-                position=stmt.start_pos,
-                search_global=True
-            )
-        elif isinstance(atom, tree.Literal):
-            string = parser_utils.safe_literal_eval(atom.value)
-            return set([compiled.create(self, string)])
-        else:
-            c = atom.children
-            if c[0].type == 'string':
-                # Will be one string.
-                types = self.eval_atom(context, c[0])
-                for string in c[1:]:
-                    right = self.eval_atom(context, string)
-                    types = precedence.calculate(self, context, types, '+', right)
-                return types
-            # Parentheses without commas are not tuples.
-            elif c[0] == '(' and not len(c) == 2 \
-                    and not(c[1].type == 'testlist_comp' and
-                            len(c[1].children) > 1):
-                return self.eval_element(context, c[1])
-
-            try:
-                comp_for = c[1].children[1]
-            except (IndexError, AttributeError):
-                pass
-            else:
-                if comp_for == ':':
-                    # Dict comprehensions have a colon at the 3rd index.
-                    try:
-                        comp_for = c[1].children[3]
-                    except IndexError:
-                        pass
-
-                if comp_for.type == 'comp_for':
-                    return set([iterable.Comprehension.from_atom(self, context, atom)])
-
-            # It's a dict/list/tuple literal.
-            array_node = c[1]
-            try:
-                array_node_c = array_node.children
-            except AttributeError:
-                array_node_c = []
-            if c[0] == '{' and (array_node == '}' or ':' in array_node_c):
-                context = iterable.DictLiteralContext(self, context, atom)
-            else:
-                context = iterable.SequenceLiteralContext(self, context, atom)
-            return set([context])
-
-    def eval_trailer(self, context, types, trailer):
-        trailer_op, node = trailer.children[:2]
-        if node == ')':  # `arglist` is optional.
-            node = ()
-
-        new_types = set()
-        if trailer_op == '[':
-            new_types |= iterable.py__getitem__(self, context, types, trailer)
-        else:
-            for typ in types:
-                debug.dbg('eval_trailer: %s in scope %s', trailer, typ)
-                if trailer_op == '.':
-                    new_types |= typ.py__getattribute__(
-                        name_context=context,
-                        name_or_str=node
-                    )
-                elif trailer_op == '(':
-                    arguments = param.TreeArguments(self, context, node, trailer)
-                    new_types |= self.execute(typ, arguments)
-        return new_types
-
-    @debug.increase_indent
-    def execute(self, obj, arguments):
-        if self.is_analysis:
-            arguments.eval_all()
-
-        debug.dbg('execute: %s %s', obj, arguments)
-        try:
-            # Some stdlib functions like super(), namedtuple(), etc. have been
-            # hard-coded in Jedi to support them.
-            return stdlib.execute(self, obj, arguments)
-        except stdlib.NotInStdLib:
-            pass
-
-        try:
-            func = obj.py__call__
-        except AttributeError:
-            debug.warning("no execution possible %s", obj)
-            return set()
-        else:
-            types = func(arguments)
-            debug.dbg('execute result: %s in %s', types, obj)
-            return types
+        return eval_node(context, element)
 
     def goto_definitions(self, context, name):
         def_ = name.get_definition(import_name_always=True)
         if def_ is not None:
             type_ = def_.type
             if type_ == 'classdef':
-                return [er.ClassContext(self, name.parent, context)]
+                return [ClassContext(self, context, name.parent)]
             elif type_ == 'funcdef':
-                return [er.FunctionContext(self, context, name.parent)]
+                return [FunctionContext.from_context(context, name.parent)]
 
             if type_ == 'expr_stmt':
                 is_simple_name = name.parent.type not in ('power', 'trailer')
                 if is_simple_name:
-                    return self.eval_statement(context, def_, name)
+                    return eval_expr_stmt(context, def_, name)
             if type_ == 'for_stmt':
-                container_types = self.eval_element(context, def_.children[3])
+                container_types = context.eval_node(def_.children[3])
                 cn = ContextualizedNode(context, def_.children[3])
-                for_types = iterable.py__iter__types(self, container_types, cn)
+                for_types = iterate_contexts(container_types, cn)
                 c_node = ContextualizedName(context, name)
-                return finder.check_tuple_assignments(self, c_node, for_types)
+                return check_tuple_assignments(self, c_node, for_types)
             if type_ in ('import_from', 'import_name'):
                 return imports.infer_import(context, name)
 
@@ -509,25 +258,27 @@ class Evaluator(object):
                 return module_names
 
         par = name.parent
-        typ = par.type
-        if typ == 'argument' and par.children[1] == '=' and par.children[0] == name:
+        node_type = par.type
+        if node_type == 'argument' and par.children[1] == '=' and par.children[0] == name:
             # Named param goto.
             trailer = par.parent
             if trailer.type == 'arglist':
                 trailer = trailer.parent
             if trailer.type != 'classdef':
                 if trailer.type == 'decorator':
-                    types = self.eval_element(context, trailer.children[1])
+                    context_set = context.eval_node(trailer.children[1])
                 else:
                     i = trailer.parent.children.index(trailer)
                     to_evaluate = trailer.parent.children[:i]
-                    types = self.eval_element(context, to_evaluate[0])
+                    if to_evaluate[0] == 'await':
+                        to_evaluate.pop(0)
+                    context_set = context.eval_node(to_evaluate[0])
                     for trailer in to_evaluate[1:]:
-                        types = self.eval_trailer(context, types, trailer)
+                        context_set = eval_trailer(context, context_set, trailer)
                 param_names = []
-                for typ in types:
+                for context in context_set:
                     try:
-                        get_param_names = typ.get_param_names
+                        get_param_names = context.get_param_names
                     except AttributeError:
                         pass
                     else:
@@ -535,18 +286,18 @@ class Evaluator(object):
                             if param_name.string_name == name.value:
                                 param_names.append(param_name)
                 return param_names
-        elif typ == 'dotted_name':  # Is a decorator.
+        elif node_type == 'dotted_name':  # Is a decorator.
             index = par.children.index(name)
             if index > 0:
                 new_dotted = helpers.deep_ast_copy(par)
                 new_dotted.children[index - 1:] = []
-                values = self.eval_element(context, new_dotted)
+                values = context.eval_node(new_dotted)
                 return unite(
                     value.py__getattribute__(name, name_context=context, is_goto=True)
                     for value in values
                 )
 
-        if typ == 'trailer' and par.children[0] == '.':
+        if node_type == 'trailer' and par.children[0] == '.':
             values = helpers.evaluate_call_of_leaf(context, name, cut_own_trailer=True)
             return unite(
                 value.py__getattribute__(name, name_context=context, is_goto=True)
@@ -589,22 +340,21 @@ class Evaluator(object):
             parent_context = from_scope_node(parent_scope, child_is_funcdef=is_funcdef)
 
             if is_funcdef:
+                func = FunctionContext.from_context(
+                    parent_context,
+                    scope_node
+                )
                 if isinstance(parent_context, AnonymousInstance):
                     func = BoundMethod(
-                        self, parent_context, parent_context.class_context,
-                        parent_context.parent_context, scope_node
-                    )
-                else:
-                    func = er.FunctionContext(
-                        self,
-                        parent_context,
-                        scope_node
+                        instance=parent_context,
+                        klass=parent_context.class_context,
+                        function=func
                     )
                 if is_nested and not node_is_object:
                     return func.get_function_execution()
                 return func
             elif scope_node.type == 'classdef':
-                class_context = er.ClassContext(self, scope_node, parent_context)
+                class_context = ClassContext(self, parent_context, scope_node)
                 if child_is_funcdef:
                     # anonymous instance
                     return AnonymousInstance(self, parent_context, class_context)
@@ -613,7 +363,7 @@ class Evaluator(object):
             elif scope_node.type == 'comp_for':
                 if node.start_pos >= scope_node.children[-1].start_pos:
                     return parent_context
-                return iterable.CompForContext.from_comp_for(parent_context, scope_node)
+                return CompForContext.from_comp_for(parent_context, scope_node)
             raise Exception("There's a scope that was not managed.")
 
         base_node = base_context.tree_node
@@ -627,3 +377,15 @@ class Evaluator(object):
                 node = node.parent
             scope_node = parent_scope(node)
         return from_scope_node(scope_node, is_nested=True, node_is_object=node_is_object)
+
+    def parse_and_get_code(self, code=None, path=None, encoding='utf-8', **kwargs):
+        if self.allow_different_encoding:
+            if code is None:
+                with open(path, 'rb') as f:
+                    code = f.read()
+            code = python_bytes_to_unicode(code, encoding=encoding, errors='replace')
+
+        return self.grammar.parse(code=code, path=path, **kwargs), code
+
+    def parse(self, *args, **kwargs):
+        return self.parse_and_get_code(*args, **kwargs)[0]

jedi/evaluate/analysis.py

@@ -1,9 +1,12 @@
 """
 Module for statical analysis.
 """
-from jedi import debug
 from parso.python import tree
+
+from jedi._compatibility import force_unicode
+from jedi import debug
 from jedi.evaluate.compiled import CompiledObject
+from jedi.evaluate.helpers import is_string
 
 
 CODES = {
@@ -92,7 +95,7 @@ def _check_for_setattr(instance):
     """
     Check if there's any setattr method inside an instance. If so, return True.
     """
-    from jedi.evaluate.representation import ModuleContext
+    from jedi.evaluate.context import ModuleContext
     module = instance.get_root_context()
     if not isinstance(module, ModuleContext):
         return False
@@ -109,14 +112,15 @@ def _check_for_setattr(instance):
 
 def add_attribute_error(name_context, lookup_context, name):
     message = ('AttributeError: %s has no attribute %s.' % (lookup_context, name))
-    from jedi.evaluate.instance import AbstractInstanceContext, CompiledInstanceName
+    from jedi.evaluate.context.instance import AbstractInstanceContext, CompiledInstanceName
     # Check for __getattr__/__getattribute__ existance and issue a warning
     # instead of an error, if that happens.
     typ = Error
     if isinstance(lookup_context, AbstractInstanceContext):
-        slot_names = lookup_context.get_function_slot_names('__getattr__') + \
-            lookup_context.get_function_slot_names('__getattribute__')
+        slot_names = lookup_context.get_function_slot_names(u'__getattr__') + \
+            lookup_context.get_function_slot_names(u'__getattribute__')
         for n in slot_names:
+            # TODO do we even get here?
             if isinstance(name, CompiledInstanceName) and \
                     n.parent_context.obj == object:
                 typ = Warning
@@ -139,7 +143,7 @@ def _check_for_exception_catch(node_context, jedi_name, exception, payload=None)
     """
     def check_match(cls, exception):
         try:
-            return isinstance(cls, CompiledObject) and issubclass(exception, cls.obj)
+            return isinstance(cls, CompiledObject) and cls.is_super_class(exception)
         except TypeError:
             return False
 
@@ -159,8 +163,8 @@ def _check_for_exception_catch(node_context, jedi_name, exception, payload=None)
             else:
                 except_classes = node_context.eval_node(node)
                 for cls in except_classes:
-                    from jedi.evaluate import iterable
-                    if isinstance(cls, iterable.AbstractSequence) and \
+                    from jedi.evaluate.context import iterable
+                    if isinstance(cls, iterable.Sequence) and \
                             cls.array_type == 'tuple':
                         # multiple exceptions
                         for lazy_context in cls.py__iter__():
@@ -181,7 +185,7 @@ def _check_for_exception_catch(node_context, jedi_name, exception, payload=None)
             assert trailer.type == 'trailer'
             arglist = trailer.children[1]
             assert arglist.type == 'arglist'
-            from jedi.evaluate.param import TreeArguments
+            from jedi.evaluate.arguments import TreeArguments
             args = list(TreeArguments(node_context.evaluator, node_context, arglist).unpack())
             # Arguments should be very simple
             assert len(args) == 2
@@ -189,8 +193,8 @@ def _check_for_exception_catch(node_context, jedi_name, exception, payload=None)
             # Check name
             key, lazy_context = args[1]
             names = list(lazy_context.infer())
-            assert len(names) == 1 and isinstance(names[0], CompiledObject)
-            assert names[0].obj == payload[1].value
+            assert len(names) == 1 and is_string(names[0])
+            assert force_unicode(names[0].get_safe_value()) == payload[1].value
 
             # Check objects
             key, lazy_context = args[0]

jedi/evaluate/arguments.py

@@ -0,0 +1,305 @@
+import re
+
+from parso.python import tree
+
+from jedi._compatibility import zip_longest
+from jedi import debug
+from jedi.evaluate import analysis
+from jedi.evaluate.lazy_context import LazyKnownContext, LazyKnownContexts, \
+    LazyTreeContext, get_merged_lazy_context
+from jedi.evaluate.filters import ParamName
+from jedi.evaluate.base_context import NO_CONTEXTS
+from jedi.evaluate.context import iterable
+from jedi.evaluate.param import get_executed_params, ExecutedParam
+
+
+def try_iter_content(types, depth=0):
+    """Helper method for static analysis."""
+    if depth > 10:
+        # It's possible that a loop has references on itself (especially with
+        # CompiledObject). Therefore don't loop infinitely.
+        return
+
+    for typ in types:
+        try:
+            f = typ.py__iter__
+        except AttributeError:
+            pass
+        else:
+            for lazy_context in f():
+                try_iter_content(lazy_context.infer(), depth + 1)
+
+
+def repack_with_argument_clinic(string, keep_arguments_param=False):
+    """
+    Transforms a function or method with arguments to the signature that is
+    given as an argument clinic notation.
+
+    Argument clinic is part of CPython and used for all the functions that are
+    implemented in C (Python 3.7):
+
+        str.split.__text_signature__
+        # Results in: '($self, /, sep=None, maxsplit=-1)'
+    """
+    clinic_args = list(_parse_argument_clinic(string))
+
+    def decorator(func):
+        def wrapper(*args, **kwargs):
+            if keep_arguments_param:
+                arguments = kwargs['arguments']
+            else:
+                arguments = kwargs.pop('arguments')
+            try:
+                args += tuple(_iterate_argument_clinic(arguments, clinic_args))
+            except ValueError:
+                return NO_CONTEXTS
+            else:
+                return func(*args, **kwargs)
+
+        return wrapper
+    return decorator
+
+
+def _iterate_argument_clinic(arguments, parameters):
+    """Uses a list with argument clinic information (see PEP 436)."""
+    iterator = arguments.unpack()
+    for i, (name, optional, allow_kwargs) in enumerate(parameters):
+        key, argument = next(iterator, (None, None))
+        if key is not None:
+            debug.warning('Keyword arguments in argument clinic are currently not supported.')
+            raise ValueError
+        if argument is None and not optional:
+            debug.warning('TypeError: %s expected at least %s arguments, got %s',
+                          name, len(parameters), i)
+            raise ValueError
+
+        context_set = NO_CONTEXTS if argument is None else argument.infer()
+
+        if not context_set and not optional:
+            # For the stdlib we always want values. If we don't get them,
+            # that's ok, maybe something is too hard to resolve, however,
+            # we will not proceed with the evaluation of that function.
+            debug.warning('argument_clinic "%s" not resolvable.', name)
+            raise ValueError
+        yield context_set
+
+
+def _parse_argument_clinic(string):
+    allow_kwargs = False
+    optional = False
+    while string:
+        # Optional arguments have to begin with a bracket. And should always be
+        # at the end of the arguments. This is therefore not a proper argument
+        # clinic implementation. `range()` for exmple allows an optional start
+        # value at the beginning.
+        match = re.match('(?:(?:(\[),? ?|, ?|)(\w+)|, ?/)\]*', string)
+        string = string[len(match.group(0)):]
+        if not match.group(2):  # A slash -> allow named arguments
+            allow_kwargs = True
+            continue
+        optional = optional or bool(match.group(1))
+        word = match.group(2)
+        yield (word, optional, allow_kwargs)
+
+
+class AbstractArguments(object):
+    context = None
+    argument_node = None
+    trailer = None
+
+    def eval_all(self, funcdef=None):
+        """
+        Evaluates all arguments as a support for static analysis
+        (normally Jedi).
+        """
+        for key, lazy_context in self.unpack():
+            types = lazy_context.infer()
+            try_iter_content(types)
+
+    def get_calling_nodes(self):
+        return []
+
+    def unpack(self, funcdef=None):
+        raise NotImplementedError
+
+    def get_executed_params(self, execution_context):
+        return get_executed_params(execution_context, self)
+
+
+class AnonymousArguments(AbstractArguments):
+    def get_executed_params(self, execution_context):
+        from jedi.evaluate.dynamic import search_params
+        return search_params(
+            execution_context.evaluator,
+            execution_context,
+            execution_context.tree_node
+        )
+
+    def __repr__(self):
+        return '%s()' % self.__class__.__name__
+
+
+class TreeArguments(AbstractArguments):
+    def __init__(self, evaluator, context, argument_node, trailer=None):
+        """
+        The argument_node is either a parser node or a list of evaluated
+        objects. Those evaluated objects may be lists of evaluated objects
+        themselves (one list for the first argument, one for the second, etc).
+
+        :param argument_node: May be an argument_node or a list of nodes.
+        """
+        self.argument_node = argument_node
+        self.context = context
+        self._evaluator = evaluator
+        self.trailer = trailer  # Can be None, e.g. in a class definition.
+
+    def _split(self):
+        if self.argument_node is None:
+            return
+
+        # Allow testlist here as well for Python2's class inheritance
+        # definitions.
+        if not (self.argument_node.type in ('arglist', 'testlist') or (
+                # in python 3.5 **arg is an argument, not arglist
+                (self.argument_node.type == 'argument') and
+                 self.argument_node.children[0] in ('*', '**'))):
+            yield 0, self.argument_node
+            return
+
+        iterator = iter(self.argument_node.children)
+        for child in iterator:
+            if child == ',':
+                continue
+            elif child in ('*', '**'):
+                yield len(child.value), next(iterator)
+            elif child.type == 'argument' and \
+                    child.children[0] in ('*', '**'):
+                assert len(child.children) == 2
+                yield len(child.children[0].value), child.children[1]
+            else:
+                yield 0, child
+
+    def unpack(self, funcdef=None):
+        named_args = []
+        for star_count, el in self._split():
+            if star_count == 1:
+                arrays = self.context.eval_node(el)
+                iterators = [_iterate_star_args(self.context, a, el, funcdef)
+                             for a in arrays]
+                for values in list(zip_longest(*iterators)):
+                    # TODO zip_longest yields None, that means this would raise
+                    # an exception?
+                    yield None, get_merged_lazy_context(
+                        [v for v in values if v is not None]
+                    )
+            elif star_count == 2:
+                arrays = self.context.eval_node(el)
+                for dct in arrays:
+                    for key, values in _star_star_dict(self.context, dct, el, funcdef):
+                        yield key, values
+            else:
+                if el.type == 'argument':
+                    c = el.children
+                    if len(c) == 3:  # Keyword argument.
+                        named_args.append((c[0].value, LazyTreeContext(self.context, c[2]),))
+                    else:  # Generator comprehension.
+                        # Include the brackets with the parent.
+                        comp = iterable.GeneratorComprehension(
+                            self._evaluator, self.context, self.argument_node.parent)
+                        yield None, LazyKnownContext(comp)
+                else:
+                    yield None, LazyTreeContext(self.context, el)
+
+        # Reordering var_args is necessary, because star args sometimes appear
+        # after named argument, but in the actual order it's prepended.
+        for named_arg in named_args:
+            yield named_arg
+
+    def as_tree_tuple_objects(self):
+        for star_count, argument in self._split():
+            if argument.type == 'argument':
+                argument, default = argument.children[::2]
+            else:
+                default = None
+            yield argument, default, star_count
+
+    def __repr__(self):
+        return '<%s: %s>' % (self.__class__.__name__, self.argument_node)
+
+    def get_calling_nodes(self):
+        from jedi.evaluate.dynamic import DynamicExecutedParams
+        old_arguments_list = []
+        arguments = self
+
+        while arguments not in old_arguments_list:
+            if not isinstance(arguments, TreeArguments):
+                break
+
+            old_arguments_list.append(arguments)
+            for name, default, star_count in reversed(list(arguments.as_tree_tuple_objects())):
+                if not star_count or not isinstance(name, tree.Name):
+                    continue
+
+                names = self._evaluator.goto(arguments.context, name)
+                if len(names) != 1:
+                    break
+                if not isinstance(names[0], ParamName):
+                    break
+                param = names[0].get_param()
+                if isinstance(param, DynamicExecutedParams):
+                    # For dynamic searches we don't even want to see errors.
+                    return []
+                if not isinstance(param, ExecutedParam):
+                    break
+                if param.var_args is None:
+                    break
+                arguments = param.var_args
+                break
+
+        if arguments.argument_node is not None:
+            return [arguments.argument_node]
+        if arguments.trailer is not None:
+            return [arguments.trailer]
+        return []
+
+
+class ValuesArguments(AbstractArguments):
+    def __init__(self, values_list):
+        self._values_list = values_list
+
+    def unpack(self, funcdef=None):
+        for values in self._values_list:
+            yield None, LazyKnownContexts(values)
+
+    def __repr__(self):
+        return '<%s: %s>' % (self.__class__.__name__, self._values_list)
+
+
+def _iterate_star_args(context, array, input_node, funcdef=None):
+    try:
+        iter_ = array.py__iter__
+    except AttributeError:
+        if funcdef is not None:
+            # TODO this funcdef should not be needed.
+            m = "TypeError: %s() argument after * must be a sequence, not %s" \
+                % (funcdef.name.value, array)
+            analysis.add(context, 'type-error-star', input_node, message=m)
+    else:
+        for lazy_context in iter_():
+            yield lazy_context
+
+
+def _star_star_dict(context, array, input_node, funcdef):
+    from jedi.evaluate.context.instance import CompiledInstance
+    if isinstance(array, CompiledInstance) and array.name.string_name == 'dict':
+        # For now ignore this case. In the future add proper iterators and just
+        # make one call without crazy isinstance checks.
+        return {}
+    elif isinstance(array, iterable.Sequence) and array.array_type == 'dict':
+        return array.exact_key_items()
+    else:
+        if funcdef is not None:
+            m = "TypeError: %s argument after ** must be a mapping, not %s" \
+                % (funcdef.name.value, array)
+            analysis.add(context, 'type-error-star-star', input_node, message=m)
+        return {}

jedi/evaluate/base_context.py

@@ -0,0 +1,279 @@
+"""
+Contexts are the "values" that Python would return. However Contexts are at the
+same time also the "contexts" that a user is currently sitting in.
+
+A ContextSet is typically used to specify the return of a function or any other
+static analysis operation. In jedi there are always multiple returns and not
+just one.
+"""
+from parso.python.tree import ExprStmt, CompFor
+
+from jedi import debug
+from jedi._compatibility import Python3Method, zip_longest, unicode
+from jedi.parser_utils import clean_scope_docstring, get_doc_with_call_signature
+from jedi.common import BaseContextSet, BaseContext
+from jedi.evaluate.helpers import EvaluatorIndexError, EvaluatorTypeError, \
+    EvaluatorKeyError
+
+
+class Context(BaseContext):
+    """
+    Should be defined, otherwise the API returns empty types.
+    """
+
+    predefined_names = {}
+    tree_node = None
+    """
+    To be defined by subclasses.
+    """
+
+    @property
+    def api_type(self):
+        # By default just lower name of the class. Can and should be
+        # overwritten.
+        return self.__class__.__name__.lower()
+
+    @debug.increase_indent
+    def execute(self, arguments):
+        """
+        In contrast to py__call__ this function is always available.
+
+        `hasattr(x, py__call__)` can also be checked to see if a context is
+        executable.
+        """
+        if self.evaluator.is_analysis:
+            arguments.eval_all()
+
+        debug.dbg('execute: %s %s', self, arguments)
+        from jedi.evaluate import stdlib
+        try:
+            # Some stdlib functions like super(), namedtuple(), etc. have been
+            # hard-coded in Jedi to support them.
+            return stdlib.execute(self.evaluator, self, arguments)
+        except stdlib.NotInStdLib:
+            pass
+
+        try:
+            func = self.py__call__
+        except AttributeError:
+            debug.warning("no execution possible %s", self)
+            return NO_CONTEXTS
+        else:
+            context_set = func(arguments)
+            debug.dbg('execute result: %s in %s', context_set, self)
+            return context_set
+
+        return self.evaluator.execute(self, arguments)
+
+    def execute_evaluated(self, *value_list):
+        """
+        Execute a function with already executed arguments.
+        """
+        from jedi.evaluate.arguments import ValuesArguments
+        arguments = ValuesArguments([ContextSet(value) for value in value_list])
+        return self.execute(arguments)
+
+    def iterate(self, contextualized_node=None, is_async=False):
+        debug.dbg('iterate %s', self)
+        try:
+            if is_async:
+                iter_method = self.py__aiter__
+            else:
+                iter_method = self.py__iter__
+        except AttributeError:
+            if contextualized_node is not None:
+                from jedi.evaluate import analysis
+                analysis.add(
+                    contextualized_node.context,
+                    'type-error-not-iterable',
+                    contextualized_node.node,
+                    message="TypeError: '%s' object is not iterable" % self)
+            return iter([])
+        else:
+            return iter_method()
+
+    def get_item(self, index_contexts, contextualized_node):
+        from jedi.evaluate.compiled import CompiledObject
+        from jedi.evaluate.context.iterable import Slice, Sequence
+        result = ContextSet()
+
+        for index in index_contexts:
+            if isinstance(index, Slice):
+                index = index.obj
+            if isinstance(index, CompiledObject):
+                try:
+                    index = index.get_safe_value()
+                except ValueError:
+                    pass
+
+            if type(index) not in (float, int, str, unicode, slice, bytes):
+                # If the index is not clearly defined, we have to get all the
+                # possiblities.
+                if isinstance(self, Sequence) and self.array_type == 'dict':
+                    result |= self.dict_values()
+                else:
+                    result |= iterate_contexts(ContextSet(self))
+                continue
+
+            # The actual getitem call.
+            try:
+                getitem = self.py__getitem__
+            except AttributeError:
+                from jedi.evaluate import analysis
+                # TODO this context is probably not right.
+                analysis.add(
+                    contextualized_node.context,
+                    'type-error-not-subscriptable',
+                    contextualized_node.node,
+                    message="TypeError: '%s' object is not subscriptable" % self
+                )
+            else:
+                try:
+                    result |= getitem(index)
+                except EvaluatorIndexError:
+                    result |= iterate_contexts(ContextSet(self))
+                except EvaluatorKeyError:
+                    # Must be a dict. Lists don't raise KeyErrors.
+                    result |= self.dict_values()
+                except EvaluatorTypeError:
+                    # The type is wrong and therefore it makes no sense to do
+                    # anything anymore.
+                    result = NO_CONTEXTS
+        return result
+
+    def eval_node(self, node):
+        return self.evaluator.eval_element(self, node)
+
+    @Python3Method
+    def py__getattribute__(self, name_or_str, name_context=None, position=None,
+                           search_global=False, is_goto=False,
+                           analysis_errors=True):
+        """
+        :param position: Position of the last statement -> tuple of line, column
+        """
+        if name_context is None:
+            name_context = self
+        from jedi.evaluate import finder
+        f = finder.NameFinder(self.evaluator, self, name_context, name_or_str,
+                              position, analysis_errors=analysis_errors)
+        filters = f.get_filters(search_global)
+        if is_goto:
+            return f.filter_name(filters)
+        return f.find(filters, attribute_lookup=not search_global)
+
+    def create_context(self, node, node_is_context=False, node_is_object=False):
+        return self.evaluator.create_context(self, node, node_is_context, node_is_object)
+
+    def is_class(self):
+        return False
+
+    def py__bool__(self):
+        """
+        Since Wrapper is a super class for classes, functions and modules,
+        the return value will always be true.
+        """
+        return True
+
+    def py__doc__(self, include_call_signature=False):
+        try:
+            self.tree_node.get_doc_node
+        except AttributeError:
+            return ''
+        else:
+            if include_call_signature:
+                return get_doc_with_call_signature(self.tree_node)
+            else:
+                return clean_scope_docstring(self.tree_node)
+        return None
+
+
+def iterate_contexts(contexts, contextualized_node=None, is_async=False):
+    """
+    Calls `iterate`, on all contexts but ignores the ordering and just returns
+    all contexts that the iterate functions yield.
+    """
+    return ContextSet.from_sets(
+        lazy_context.infer()
+        for lazy_context in contexts.iterate(contextualized_node, is_async=is_async)
+    )
+
+
+class TreeContext(Context):
+    def __init__(self, evaluator, parent_context, tree_node):
+        super(TreeContext, self).__init__(evaluator, parent_context)
+        self.predefined_names = {}
+        self.tree_node = tree_node
+
+    def __repr__(self):
+        return '<%s: %s>' % (self.__class__.__name__, self.tree_node)
+
+
+class ContextualizedNode(object):
+    def __init__(self, context, node):
+        self.context = context
+        self.node = node
+
+    def get_root_context(self):
+        return self.context.get_root_context()
+
+    def infer(self):
+        return self.context.eval_node(self.node)
+
+
+class ContextualizedName(ContextualizedNode):
+    # TODO merge with TreeNameDefinition?!
+    @property
+    def name(self):
+        return self.node
+
+    def assignment_indexes(self):
+        """
+        Returns an array of tuple(int, node) of the indexes that are used in
+        tuple assignments.
+
+        For example if the name is ``y`` in the following code::
+
+            x, (y, z) = 2, ''
+
+        would result in ``[(1, xyz_node), (0, yz_node)]``.
+        """
+        indexes = []
+        node = self.node.parent
+        compare = self.node
+        while node is not None:
+            if node.type in ('testlist', 'testlist_comp', 'testlist_star_expr', 'exprlist'):
+                for i, child in enumerate(node.children):
+                    if child == compare:
+                        indexes.insert(0, (int(i / 2), node))
+                        break
+                else:
+                    raise LookupError("Couldn't find the assignment.")
+            elif isinstance(node, (ExprStmt, CompFor)):
+                break
+
+            compare = node
+            node = node.parent
+        return indexes
+
+
+class ContextSet(BaseContextSet):
+    def py__class__(self):
+        return ContextSet.from_iterable(c.py__class__() for c in self._set)
+
+    def iterate(self, contextualized_node=None, is_async=False):
+        from jedi.evaluate.lazy_context import get_merged_lazy_context
+        type_iters = [c.iterate(contextualized_node, is_async=is_async) for c in self._set]
+        for lazy_contexts in zip_longest(*type_iters):
+            yield get_merged_lazy_context(
+                [l for l in lazy_contexts if l is not None]
+            )
+
+
+NO_CONTEXTS = ContextSet()
+
+
+def iterator_to_context_set(func):
+    def wrapper(*args, **kwargs):
+        return ContextSet.from_iterable(func(*args, **kwargs))
+
+    return wrapper

jedi/evaluate/cache.py

@@ -4,8 +4,6 @@
 - ``CachedMetaClass`` uses ``_memoize_default`` to do the same with classes.
 """
 
-import inspect
-
 _NO_DEFAULT = object()
 
 
@@ -40,8 +38,6 @@ def _memoize_default(default=_NO_DEFAULT, evaluator_is_first_arg=False, second_a
                 if default is not _NO_DEFAULT:
                     memo[key] = default
                 rv = function(obj, *args, **kwargs)
-                if inspect.isgenerator(rv):
-                    rv = list(rv)
                 memo[key] = rv
                 return rv
         return wrapper
@@ -63,7 +59,7 @@ def evaluator_method_cache(default=_NO_DEFAULT):
     return decorator
 
 
-def _memoize_meta_class():
+def evaluator_as_method_param_cache():
     def decorator(call):
         return _memoize_default(second_arg_is_evaluator=True)(call)
 
@@ -76,6 +72,6 @@ class CachedMetaClass(type):
     class initializations. Either you do it this way or with decorators, but
     with decorators you lose class access (isinstance, etc).
     """
-    @_memoize_meta_class()
+    @evaluator_as_method_param_cache()
     def __call__(self, *args, **kwargs):
         return super(CachedMetaClass, self).__call__(*args, **kwargs)

jedi/evaluate/compiled/__init__.py

@@ -1,637 +1,43 @@
-"""
-Imitate the parser representation.
-"""
-import inspect
-import re
-import sys
-import os
-import types
-from functools import partial
-
-from jedi._compatibility import builtins as _builtins, unicode, py_version
-from jedi import debug
-from jedi.cache import underscore_memoization, memoize_method
-from jedi.evaluate.filters import AbstractFilter, AbstractNameDefinition, \
-    ContextNameMixin
-from jedi.evaluate.context import Context, LazyKnownContext
-from jedi.evaluate.compiled.getattr_static import getattr_static
-from . import fake
-
-
-_sep = os.path.sep
-if os.path.altsep is not None:
-    _sep += os.path.altsep
-_path_re = re.compile('(?:\.[^{0}]+|[{0}]__init__\.py)$'.format(re.escape(_sep)))
-del _sep
-
-# Those types don't exist in typing.
-MethodDescriptorType = type(str.replace)
-WrapperDescriptorType = type(set.__iter__)
-# `object.__subclasshook__` is an already executed descriptor.
-object_class_dict = type.__dict__["__dict__"].__get__(object)
-ClassMethodDescriptorType = type(object_class_dict['__subclasshook__'])
-
-ALLOWED_DESCRIPTOR_ACCESS = (
-    types.FunctionType,
-    types.GetSetDescriptorType,
-    types.MemberDescriptorType,
-    MethodDescriptorType,
-    WrapperDescriptorType,
-    ClassMethodDescriptorType,
-    staticmethod,
-    classmethod,
-)
-
-class CheckAttribute(object):
-    """Raises an AttributeError if the attribute X isn't available."""
-    def __init__(self, func):
-        self.func = func
-        # Remove the py in front of e.g. py__call__.
-        self.check_name = func.__name__[2:]
-
-    def __get__(self, instance, owner):
-        # This might raise an AttributeError. That's wanted.
-        if self.check_name == '__iter__':
-            # Python iterators are a bit strange, because there's no need for
-            # the __iter__ function as long as __getitem__ is defined (it will
-            # just start with __getitem__(0). This is especially true for
-            # Python 2 strings, where `str.__iter__` is not even defined.
-            try:
-                iter(instance.obj)
-            except TypeError:
-                raise AttributeError
-        else:
-            getattr(instance.obj, self.check_name)
-        return partial(self.func, instance)
-
-
-class CompiledObject(Context):
-    path = None  # modules have this attribute - set it to None.
-    used_names = lambda self: {}  # To be consistent with modules.
-
-    def __init__(self, evaluator, obj, parent_context=None, faked_class=None):
-        super(CompiledObject, self).__init__(evaluator, parent_context)
-        self.obj = obj
-        # This attribute will not be set for most classes, except for fakes.
-        self.tree_node = faked_class
-
-    def get_root_node(self):
-        # To make things a bit easier with filters we add this method here.
-        return self.get_root_context()
-
-    @CheckAttribute
-    def py__call__(self, params):
-        if inspect.isclass(self.obj):
-            from jedi.evaluate.instance import CompiledInstance
-            return set([CompiledInstance(self.evaluator, self.parent_context, self, params)])
-        else:
-            return set(self._execute_function(params))
-
-    @CheckAttribute
-    def py__class__(self):
-        return create(self.evaluator, self.obj.__class__)
-
-    @CheckAttribute
-    def py__mro__(self):
-        return (self,) + tuple(create(self.evaluator, cls) for cls in self.obj.__mro__[1:])
-
-    @CheckAttribute
-    def py__bases__(self):
-        return tuple(create(self.evaluator, cls) for cls in self.obj.__bases__)
-
-    def py__bool__(self):
-        return bool(self.obj)
-
-    def py__file__(self):
-        try:
-            return self.obj.__file__
-        except AttributeError:
-            return None
-
-    def is_class(self):
-        return inspect.isclass(self.obj)
-
-    def py__doc__(self, include_call_signature=False):
-        return inspect.getdoc(self.obj) or ''
-
-    def get_param_names(self):
-        obj = self.obj
-        try:
-            if py_version < 33:
-                raise ValueError("inspect.signature was introduced in 3.3")
-            if py_version == 34:
-                # In 3.4 inspect.signature are wrong for str and int. This has
-                # been fixed in 3.5. The signature of object is returned,
-                # because no signature was found for str. Here we imitate 3.5
-                # logic and just ignore the signature if the magic methods
-                # don't match object.
-                # 3.3 doesn't even have the logic and returns nothing for str
-                # and classes that inherit from object.
-                user_def = inspect._signature_get_user_defined_method
-                if (inspect.isclass(obj)
-                        and not user_def(type(obj), '__init__')
-                        and not user_def(type(obj), '__new__')
-                        and (obj.__init__ != object.__init__
-                             or obj.__new__ != object.__new__)):
-                    raise ValueError
-
-            signature = inspect.signature(obj)
-        except ValueError:  # Has no signature
-            params_str, ret = self._parse_function_doc()
-            tokens = params_str.split(',')
-            if inspect.ismethoddescriptor(obj):
-                tokens.insert(0, 'self')
-            for p in tokens:
-                parts = p.strip().split('=')
-                yield UnresolvableParamName(self, parts[0])
-        else:
-            for signature_param in signature.parameters.values():
-                yield SignatureParamName(self, signature_param)
-
-    def __repr__(self):
-        return '<%s: %s>' % (self.__class__.__name__, repr(self.obj))
-
-    @underscore_memoization
-    def _parse_function_doc(self):
-        doc = self.py__doc__()
-        if doc is None:
-            return '', ''
-
-        return _parse_function_doc(doc)
-
-    @property
-    def api_type(self):
-        obj = self.obj
-        if inspect.isclass(obj):
-            return 'class'
-        elif inspect.ismodule(obj):
-            return 'module'
-        elif inspect.isbuiltin(obj) or inspect.ismethod(obj) \
-                or inspect.ismethoddescriptor(obj) or inspect.isfunction(obj):
-            return 'function'
-        # Everything else...
-        return 'instance'
-
-    @property
-    def type(self):
-        """Imitate the tree.Node.type values."""
-        cls = self._get_class()
-        if inspect.isclass(cls):
-            return 'classdef'
-        elif inspect.ismodule(cls):
-            return 'file_input'
-        elif inspect.isbuiltin(cls) or inspect.ismethod(cls) or \
-                inspect.ismethoddescriptor(cls):
-            return 'funcdef'
-
-    @underscore_memoization
-    def _cls(self):
-        """
-        We used to limit the lookups for instantiated objects like list(), but
-        this is not the case anymore. Python itself
-        """
-        # Ensures that a CompiledObject is returned that is not an instance (like list)
-        return self
-
-    def _get_class(self):
-        if not fake.is_class_instance(self.obj) or \
-                inspect.ismethoddescriptor(self.obj):  # slots
-            return self.obj
-
-        try:
-            return self.obj.__class__
-        except AttributeError:
-            # happens with numpy.core.umath._UFUNC_API (you get it
-            # automatically by doing `import numpy`.
-            return type
-
-    def get_filters(self, search_global=False, is_instance=False,
-                    until_position=None, origin_scope=None):
-        yield self._ensure_one_filter(is_instance)
-
-    @memoize_method
-    def _ensure_one_filter(self, is_instance):
-        """
-        search_global shouldn't change the fact that there's one dict, this way
-        there's only one `object`.
-        """
-        return CompiledObjectFilter(self.evaluator, self, is_instance)
-
-    @CheckAttribute
-    def py__getitem__(self, index):
-        if type(self.obj) not in (str, list, tuple, unicode, bytes, bytearray, dict):
-            # Get rid of side effects, we won't call custom `__getitem__`s.
-            return set()
-
-        return set([create(self.evaluator, self.obj[index])])
-
-    @CheckAttribute
-    def py__iter__(self):
-        if type(self.obj) not in (str, list, tuple, unicode, bytes, bytearray, dict):
-            # Get rid of side effects, we won't call custom `__getitem__`s.
-            return
-
-        for i, part in enumerate(self.obj):
-            if i > 20:
-                # Should not go crazy with large iterators
-                break
-            yield LazyKnownContext(create(self.evaluator, part))
-
-    def py__name__(self):
-        try:
-            return self._get_class().__name__
-        except AttributeError:
-            return None
-
-    @property
-    def name(self):
-        try:
-            name = self._get_class().__name__
-        except AttributeError:
-            name = repr(self.obj)
-        return CompiledContextName(self, name)
-
-    def _execute_function(self, params):
-        from jedi.evaluate import docstrings
-        if self.type != 'funcdef':
-            return
-        for name in self._parse_function_doc()[1].split():
-            try:
-                bltn_obj = getattr(_builtins, name)
-            except AttributeError:
-                continue
-            else:
-                if bltn_obj is None:
-                    # We want to evaluate everything except None.
-                    # TODO do we?
-                    continue
-                bltn_obj = create(self.evaluator, bltn_obj)
-                for result in self.evaluator.execute(bltn_obj, params):
-                    yield result
-        for type_ in docstrings.infer_return_types(self):
-            yield type_
-
-    def get_self_attributes(self):
-        return []  # Instance compatibility
-
-    def get_imports(self):
-        return []  # Builtins don't have imports
-
-    def dict_values(self):
-        return set(create(self.evaluator, v) for v in self.obj.values())
-
-
-class CompiledName(AbstractNameDefinition):
-    def __init__(self, evaluator, parent_context, name):
-        self._evaluator = evaluator
-        self.parent_context = parent_context
-        self.string_name = name
-
-    def __repr__(self):
-        try:
-            name = self.parent_context.name  # __name__ is not defined all the time
-        except AttributeError:
-            name = None
-        return '<%s: (%s).%s>' % (self.__class__.__name__, name, self.string_name)
-
-    @property
-    def api_type(self):
-        return next(iter(self.infer())).api_type
-
-    @underscore_memoization
-    def infer(self):
-        module = self.parent_context.get_root_context()
-        return [_create_from_name(self._evaluator, module, self.parent_context, self.string_name)]
-
-
-class SignatureParamName(AbstractNameDefinition):
-    api_type = 'param'
-
-    def __init__(self, compiled_obj, signature_param):
-        self.parent_context = compiled_obj.parent_context
-        self._signature_param = signature_param
-
-    @property
-    def string_name(self):
-        return self._signature_param.name
-
-    def infer(self):
-        p = self._signature_param
-        evaluator = self.parent_context.evaluator
-        types = set()
-        if p.default is not p.empty:
-            types.add(create(evaluator, p.default))
-        if p.annotation is not p.empty:
-            annotation = create(evaluator, p.annotation)
-            types |= annotation.execute_evaluated()
-        return types
-
-
-class UnresolvableParamName(AbstractNameDefinition):
-    api_type = 'param'
-
-    def __init__(self, compiled_obj, name):
-        self.parent_context = compiled_obj.parent_context
-        self.string_name = name
-
-    def infer(self):
-        return set()
-
-
-class CompiledContextName(ContextNameMixin, AbstractNameDefinition):
-    def __init__(self, context, name):
-        self.string_name = name
-        self._context = context
-        self.parent_context = context.parent_context
-
-
-class EmptyCompiledName(AbstractNameDefinition):
-    """
-    Accessing some names will raise an exception. To avoid not having any
-    completions, just give Jedi the option to return this object. It infers to
-    nothing.
-    """
-    def __init__(self, evaluator, name):
-        self.parent_context = evaluator.BUILTINS
-        self.string_name = name
-
-    def infer(self):
-        return []
-
-
-class CompiledObjectFilter(AbstractFilter):
-    name_class = CompiledName
-
-    def __init__(self, evaluator, compiled_object, is_instance=False):
-        self._evaluator = evaluator
-        self._compiled_object = compiled_object
-        self._is_instance = is_instance
-
-    @memoize_method
-    def get(self, name):
-        name = str(name)
-        obj = self._compiled_object.obj
-        try:
-            attr, is_get_descriptor = getattr_static(obj, name)
-        except AttributeError:
-            return []
-        else:
-            if is_get_descriptor \
-                    and not type(attr) in ALLOWED_DESCRIPTOR_ACCESS:
-                # In case of descriptors that have get methods we cannot return
-                # it's value, because that would mean code execution.
-                return [EmptyCompiledName(self._evaluator, name)]
-            if self._is_instance and name not in dir(obj):
-                return []
-        return [self._create_name(name)]
-
-    def values(self):
-        obj = self._compiled_object.obj
-
-        names = []
-        for name in dir(obj):
-            names += self.get(name)
-
-        is_instance = self._is_instance or fake.is_class_instance(obj)
-        # ``dir`` doesn't include the type names.
-        if not inspect.ismodule(obj) and (obj is not type) and not is_instance:
-            for filter in create(self._evaluator, type).get_filters():
-                names += filter.values()
-        return names
-
-    def _create_name(self, name):
-        return self.name_class(self._evaluator, self._compiled_object, name)
-
-
-def dotted_from_fs_path(fs_path, sys_path):
-    """
-    Changes `/usr/lib/python3.4/email/utils.py` to `email.utils`.  I.e.
-    compares the path with sys.path and then returns the dotted_path. If the
-    path is not in the sys.path, just returns None.
-    """
-    if os.path.basename(fs_path).startswith('__init__.'):
-        # We are calculating the path. __init__ files are not interesting.
-        fs_path = os.path.dirname(fs_path)
-
-    # prefer
-    #   - UNIX
-    #     /path/to/pythonX.Y/lib-dynload
-    #     /path/to/pythonX.Y/site-packages
-    #   - Windows
-    #     C:\path\to\DLLs
-    #     C:\path\to\Lib\site-packages
-    # over
-    #   - UNIX
-    #     /path/to/pythonX.Y
-    #   - Windows
-    #     C:\path\to\Lib
-    path = ''
-    for s in sys_path:
-        if (fs_path.startswith(s) and len(path) < len(s)):
-            path = s
-
-    # - Window
-    # X:\path\to\lib-dynload/datetime.pyd => datetime
-    module_path = fs_path[len(path):].lstrip(os.path.sep).lstrip('/')
-    # - Window
-    # Replace like X:\path\to\something/foo/bar.py
-    return _path_re.sub('', module_path).replace(os.path.sep, '.').replace('/', '.')
-
-
-def load_module(evaluator, path=None, name=None):
-    sys_path = evaluator.sys_path
-    if path is not None:
-        dotted_path = dotted_from_fs_path(path, sys_path=sys_path)
-    else:
-        dotted_path = name
-
-    if dotted_path is None:
-        p, _, dotted_path = path.partition(os.path.sep)
-        sys_path.insert(0, p)
-
-    temp, sys.path = sys.path, sys_path
-    try:
-        __import__(dotted_path)
-    except RuntimeError:
-        if 'PySide' in dotted_path or 'PyQt' in dotted_path:
-            # RuntimeError: the PyQt4.QtCore and PyQt5.QtCore modules both wrap
-            # the QObject class.
-            # See https://github.com/davidhalter/jedi/pull/483
-            return None
-        raise
-    except ImportError:
-        # If a module is "corrupt" or not really a Python module or whatever.
-        debug.warning('Module %s not importable in path %s.', dotted_path, path)
-        return None
-    finally:
-        sys.path = temp
-
-    # Just access the cache after import, because of #59 as well as the very
-    # complicated import structure of Python.
-    module = sys.modules[dotted_path]
-
-    return create(evaluator, module)
-
-
-docstr_defaults = {
-    'floating point number': 'float',
-    'character': 'str',
-    'integer': 'int',
-    'dictionary': 'dict',
-    'string': 'str',
-}
-
-
-def _parse_function_doc(doc):
-    """
-    Takes a function and returns the params and return value as a tuple.
-    This is nothing more than a docstring parser.
-
-    TODO docstrings like utime(path, (atime, mtime)) and a(b [, b]) -> None
-    TODO docstrings like 'tuple of integers'
-    """
-    # parse round parentheses: def func(a, (b,c))
-    try:
-        count = 0
-        start = doc.index('(')
-        for i, s in enumerate(doc[start:]):
-            if s == '(':
-                count += 1
-            elif s == ')':
-                count -= 1
-            if count == 0:
-                end = start + i
-                break
-        param_str = doc[start + 1:end]
-    except (ValueError, UnboundLocalError):
-        # ValueError for doc.index
-        # UnboundLocalError for undefined end in last line
-        debug.dbg('no brackets found - no param')
-        end = 0
-        param_str = ''
-    else:
-        # remove square brackets, that show an optional param ( = None)
-        def change_options(m):
-            args = m.group(1).split(',')
-            for i, a in enumerate(args):
-                if a and '=' not in a:
-                    args[i] += '=None'
-            return ','.join(args)
-
-        while True:
-            param_str, changes = re.subn(r' ?\[([^\[\]]+)\]',
-                                         change_options, param_str)
-            if changes == 0:
-                break
-    param_str = param_str.replace('-', '_')  # see: isinstance.__doc__
-
-    # parse return value
-    r = re.search('-[>-]* ', doc[end:end + 7])
-    if r is None:
-        ret = ''
-    else:
-        index = end + r.end()
-        # get result type, which can contain newlines
-        pattern = re.compile(r'(,\n|[^\n-])+')
-        ret_str = pattern.match(doc, index).group(0).strip()
-        # New object -> object()
-        ret_str = re.sub(r'[nN]ew (.*)', r'\1()', ret_str)
-
-        ret = docstr_defaults.get(ret_str, ret_str)
-
-    return param_str, ret
-
-
-def _create_from_name(evaluator, module, compiled_object, name):
-    obj = compiled_object.obj
-    faked = None
-    try:
-        faked = fake.get_faked(evaluator, module, obj, parent_context=compiled_object, name=name)
-        if faked.type == 'funcdef':
-            from jedi.evaluate.representation import FunctionContext
-            return FunctionContext(evaluator, compiled_object, faked)
-    except fake.FakeDoesNotExist:
-        pass
-
-    try:
-        obj = getattr(obj, name)
-    except AttributeError:
-        # Happens e.g. in properties of
-        # PyQt4.QtGui.QStyleOptionComboBox.currentText
-        # -> just set it to None
-        obj = None
-    return create(evaluator, obj, parent_context=compiled_object, faked=faked)
+from jedi._compatibility import unicode
+from jedi.evaluate.compiled.context import CompiledObject, CompiledName, \
+    CompiledObjectFilter, CompiledContextName, create_from_access_path, \
+    create_from_name
 
 
 def builtin_from_name(evaluator, string):
-    bltn_obj = getattr(_builtins, string)
-    return create(evaluator, bltn_obj)
+    builtins = evaluator.builtins_module
+    return create_from_name(evaluator, builtins, string)
 
 
-def _a_generator(foo):
-    """Used to have an object to return for generators."""
-    yield 42
-    yield foo
-
-
-_SPECIAL_OBJECTS = {
-    'FUNCTION_CLASS': type(load_module),
-    'METHOD_CLASS': type(CompiledObject.is_class),
-    'MODULE_CLASS': type(os),
-    'GENERATOR_OBJECT': _a_generator(1.0),
-    'BUILTINS': _builtins,
-}
+def create_simple_object(evaluator, obj):
+    """
+    Only allows creations of objects that are easily picklable across Python
+    versions.
+    """
+    assert isinstance(obj, (int, float, str, bytes, unicode, slice, complex))
+    return create_from_access_path(
+        evaluator,
+        evaluator.compiled_subprocess.create_simple_object(obj)
+    )
 
 
 def get_special_object(evaluator, identifier):
-    obj = _SPECIAL_OBJECTS[identifier]
-    return create(evaluator, obj, parent_context=create(evaluator, _builtins))
+    return create_from_access_path(
+        evaluator,
+        evaluator.compiled_subprocess.get_special_object(identifier)
+    )
 
 
-def compiled_objects_cache(attribute_name):
-    def decorator(func):
-        """
-        This decorator caches just the ids, oopposed to caching the object itself.
-        Caching the id has the advantage that an object doesn't need to be
-        hashable.
-        """
-        def wrapper(evaluator, obj, parent_context=None, module=None, faked=None):
-            cache = getattr(evaluator, attribute_name)
-            # Do a very cheap form of caching here.
-            key = id(obj), id(parent_context)
-            try:
-                return cache[key][0]
-            except KeyError:
-                # TODO this whole decorator is way too ugly
-                result = func(evaluator, obj, parent_context, module, faked)
-                # Need to cache all of them, otherwise the id could be overwritten.
-                cache[key] = result, obj, parent_context, module, faked
-                return result
-        return wrapper
-
-    return decorator
+def get_string_context_set(evaluator):
+    return builtin_from_name(evaluator, u'str').execute_evaluated()
 
 
-@compiled_objects_cache('compiled_cache')
-def create(evaluator, obj, parent_context=None, module=None, faked=None):
-    """
-    A very weird interface class to this module. The more options provided the
-    more acurate loading compiled objects is.
-    """
-    if inspect.ismodule(obj):
-        if parent_context is not None:
-            # Modules don't have parents, be careful with caching: recurse.
-            return create(evaluator, obj)
-    else:
-        if parent_context is None and obj is not _builtins:
-            return create(evaluator, obj, create(evaluator, _builtins))
-
-        try:
-            faked = fake.get_faked(evaluator, module, obj, parent_context=parent_context)
-            if faked.type == 'funcdef':
-                from jedi.evaluate.representation import FunctionContext
-                return FunctionContext(evaluator, parent_context, faked)
-        except fake.FakeDoesNotExist:
-            pass
-
-    return CompiledObject(evaluator, obj, parent_context, faked)
+def load_module(evaluator, dotted_name, **kwargs):
+    # Temporary, some tensorflow builtins cannot be loaded, so it's tried again
+    # and again and it's really slow.
+    if dotted_name.startswith('tensorflow.'):
+        return None
+    access_path = evaluator.compiled_subprocess.load_module(dotted_name=dotted_name, **kwargs)
+    if access_path is None:
+        return None
+    return create_from_access_path(evaluator, access_path)

jedi/evaluate/compiled/access.py

@@ -0,0 +1,483 @@
+import inspect
+import types
+import sys
+from textwrap import dedent
+import operator as op
+from collections import namedtuple
+
+from jedi._compatibility import unicode, is_py3, builtins, \
+    py_version, force_unicode, print_to_stderr
+from jedi.evaluate.compiled.getattr_static import getattr_static
+
+
+MethodDescriptorType = type(str.replace)
+# These are not considered classes and access is granted even though they have
+# a __class__ attribute.
+NOT_CLASS_TYPES = (
+    types.BuiltinFunctionType,
+    types.CodeType,
+    types.FrameType,
+    types.FunctionType,
+    types.GeneratorType,
+    types.GetSetDescriptorType,
+    types.LambdaType,
+    types.MemberDescriptorType,
+    types.MethodType,
+    types.ModuleType,
+    types.TracebackType,
+    MethodDescriptorType
+)
+
+if is_py3:
+    NOT_CLASS_TYPES += (
+        types.MappingProxyType,
+        types.SimpleNamespace,
+        types.DynamicClassAttribute,
+    )
+
+
+# Those types don't exist in typing.
+MethodDescriptorType = type(str.replace)
+WrapperDescriptorType = type(set.__iter__)
+# `object.__subclasshook__` is an already executed descriptor.
+object_class_dict = type.__dict__["__dict__"].__get__(object)
+ClassMethodDescriptorType = type(object_class_dict['__subclasshook__'])
+
+def _a_generator(foo):
+    """Used to have an object to return for generators."""
+    yield 42
+    yield foo
+
+
+_sentinel = object()
+
+# Maps Python syntax to the operator module.
+COMPARISON_OPERATORS = {
+    '==': op.eq,
+    '!=': op.ne,
+    'is': op.is_,
+    'is not': op.is_not,
+    '<': op.lt,
+    '<=': op.le,
+    '>': op.gt,
+    '>=': op.ge,
+}
+
+_OPERATORS = {
+    '+': op.add,
+    '-': op.sub,
+}
+_OPERATORS.update(COMPARISON_OPERATORS)
+
+ALLOWED_DESCRIPTOR_ACCESS = (
+    types.FunctionType,
+    types.GetSetDescriptorType,
+    types.MemberDescriptorType,
+    MethodDescriptorType,
+    WrapperDescriptorType,
+    ClassMethodDescriptorType,
+    staticmethod,
+    classmethod,
+)
+
+
+def safe_getattr(obj, name, default=_sentinel):
+    try:
+        attr, is_get_descriptor = getattr_static(obj, name)
+    except AttributeError:
+        if default is _sentinel:
+            raise
+        return default
+    else:
+        if type(attr) in ALLOWED_DESCRIPTOR_ACCESS:
+            # In case of descriptors that have get methods we cannot return
+            # it's value, because that would mean code execution.
+            return getattr(obj, name)
+    return attr
+
+
+SignatureParam = namedtuple(
+    'SignatureParam',
+    'name has_default default has_annotation annotation kind_name'
+)
+
+
+def compiled_objects_cache(attribute_name):
+    def decorator(func):
+        """
+        This decorator caches just the ids, oopposed to caching the object itself.
+        Caching the id has the advantage that an object doesn't need to be
+        hashable.
+        """
+        def wrapper(evaluator, obj, parent_context=None):
+            cache = getattr(evaluator, attribute_name)
+            # Do a very cheap form of caching here.
+            key = id(obj)
+            try:
+                cache[key]
+                return cache[key][0]
+            except KeyError:
+                # TODO wuaaaarrghhhhhhhh
+                if attribute_name == 'mixed_cache':
+                    result = func(evaluator, obj, parent_context)
+                else:
+                    result = func(evaluator, obj)
+                # Need to cache all of them, otherwise the id could be overwritten.
+                cache[key] = result, obj, parent_context
+                return result
+        return wrapper
+
+    return decorator
+
+
+def create_access(evaluator, obj):
+    return evaluator.compiled_subprocess.get_or_create_access_handle(obj)
+
+
+def load_module(evaluator, dotted_name, sys_path):
+    temp, sys.path = sys.path, sys_path
+    try:
+        __import__(dotted_name)
+    except ImportError:
+        # If a module is "corrupt" or not really a Python module or whatever.
+        print_to_stderr('Module %s not importable in path %s.' % (dotted_name, sys_path))
+        return None
+    except Exception:
+        # Since __import__ pretty much makes code execution possible, just
+        # catch any error here and print it.
+        import traceback
+        print_to_stderr("Cannot import:\n%s" % traceback.format_exc())
+        return None
+    finally:
+        sys.path = temp
+
+    # Just access the cache after import, because of #59 as well as the very
+    # complicated import structure of Python.
+    module = sys.modules[dotted_name]
+    return create_access_path(evaluator, module)
+
+
+class AccessPath(object):
+    def __init__(self, accesses):
+        self.accesses = accesses
+
+    # Writing both of these methods here looks a bit ridiculous. However with
+    # the differences of Python 2/3 it's actually necessary, because we will
+    # otherwise have a accesses attribute that is bytes instead of unicode.
+    def __getstate__(self):
+        return self.accesses
+
+    def __setstate__(self, value):
+        self.accesses = value
+
+
+def create_access_path(evaluator, obj):
+    access = create_access(evaluator, obj)
+    return AccessPath(access.get_access_path_tuples())
+
+
+def _force_unicode_decorator(func):
+    return lambda *args, **kwargs: force_unicode(func(*args, **kwargs))
+
+
+class DirectObjectAccess(object):
+    def __init__(self, evaluator, obj):
+        self._evaluator = evaluator
+        self._obj = obj
+
+    def __repr__(self):
+        return '%s(%s)' % (self.__class__.__name__, self.get_repr())
+
+    def _create_access(self, obj):
+        return create_access(self._evaluator, obj)
+
+    def _create_access_path(self, obj):
+        return create_access_path(self._evaluator, obj)
+
+    def py__bool__(self):
+        return bool(self._obj)
+
+    def py__file__(self):
+        try:
+            return self._obj.__file__
+        except AttributeError:
+            return None
+
+    def py__doc__(self, include_call_signature=False):
+        return force_unicode(inspect.getdoc(self._obj)) or u''
+
+    def py__name__(self):
+        if not _is_class_instance(self._obj) or \
+                inspect.ismethoddescriptor(self._obj):  # slots
+            cls = self._obj
+        else:
+            try:
+                cls = self._obj.__class__
+            except AttributeError:
+                # happens with numpy.core.umath._UFUNC_API (you get it
+                # automatically by doing `import numpy`.
+                return None
+
+        try:
+            return force_unicode(cls.__name__)
+        except AttributeError:
+            return None
+
+    def py__mro__accesses(self):
+        return tuple(self._create_access_path(cls) for cls in self._obj.__mro__[1:])
+
+    def py__getitem__(self, index):
+        if type(self._obj) not in (str, list, tuple, unicode, bytes, bytearray, dict):
+            # Get rid of side effects, we won't call custom `__getitem__`s.
+            return None
+
+        return self._create_access_path(self._obj[index])
+
+    def py__iter__list(self):
+        if type(self._obj) not in (str, list, tuple, unicode, bytes, bytearray, dict):
+            # Get rid of side effects, we won't call custom `__getitem__`s.
+            return []
+
+        lst = []
+        for i, part in enumerate(self._obj):
+            if i > 20:
+                # Should not go crazy with large iterators
+                break
+            lst.append(self._create_access_path(part))
+        return lst
+
+    def py__class__(self):
+        return self._create_access_path(self._obj.__class__)
+
+    def py__bases__(self):
+        return [self._create_access_path(base) for base in self._obj.__bases__]
+
+    def py__path__(self):
+        return self._obj.__path__
+
+    @_force_unicode_decorator
+    def get_repr(self):
+        builtins = 'builtins', '__builtin__'
+
+        if inspect.ismodule(self._obj):
+            return repr(self._obj)
+        # Try to avoid execution of the property.
+        if safe_getattr(self._obj, '__module__', default='') in builtins:
+            return repr(self._obj)
+
+        type_ = type(self._obj)
+        if type_ == type:
+            return type.__repr__(self._obj)
+
+        if safe_getattr(type_, '__module__', default='') in builtins:
+            # Allow direct execution of repr for builtins.
+            return repr(self._obj)
+        return object.__repr__(self._obj)
+
+    def is_class(self):
+        return inspect.isclass(self._obj)
+
+    def ismethoddescriptor(self):
+        return inspect.ismethoddescriptor(self._obj)
+
+    def dir(self):
+        return list(map(force_unicode, dir(self._obj)))
+
+    def has_iter(self):
+        try:
+            iter(self._obj)
+            return True
+        except TypeError:
+            return False
+
+    def is_allowed_getattr(self, name):
+        # TODO this API is ugly.
+        try:
+            attr, is_get_descriptor = getattr_static(self._obj, name)
+        except AttributeError:
+            return False, False
+        else:
+            if is_get_descriptor and type(attr) not in ALLOWED_DESCRIPTOR_ACCESS:
+                # In case of descriptors that have get methods we cannot return
+                # it's value, because that would mean code execution.
+                return True, True
+        return True, False
+
+    def getattr(self, name, default=_sentinel):
+        try:
+            return self._create_access(getattr(self._obj, name))
+        except AttributeError:
+            # Happens e.g. in properties of
+            # PyQt4.QtGui.QStyleOptionComboBox.currentText
+            # -> just set it to None
+            if default is _sentinel:
+                raise
+            return self._create_access(default)
+
+    def get_safe_value(self):
+        if type(self._obj) in (bool, bytes, float, int, str, unicode, slice):
+            return self._obj
+        raise ValueError("Object is type %s and not simple" % type(self._obj))
+
+    def get_api_type(self):
+        obj = self._obj
+        if self.is_class():
+            return u'class'
+        elif inspect.ismodule(obj):
+            return u'module'
+        elif inspect.isbuiltin(obj) or inspect.ismethod(obj) \
+                or inspect.ismethoddescriptor(obj) or inspect.isfunction(obj):
+            return u'function'
+        # Everything else...
+        return u'instance'
+
+    def get_access_path_tuples(self):
+        accesses = [create_access(self._evaluator, o) for o in self._get_objects_path()]
+        return [(access.py__name__(), access) for access in accesses]
+
+    def _get_objects_path(self):
+        def get():
+            obj = self._obj
+            yield obj
+            try:
+                obj = obj.__objclass__
+            except AttributeError:
+                pass
+            else:
+                yield obj
+
+            try:
+                # Returns a dotted string path.
+                imp_plz = obj.__module__
+            except AttributeError:
+                # Unfortunately in some cases like `int` there's no __module__
+                if not inspect.ismodule(obj):
+                    yield builtins
+            else:
+                if imp_plz is None:
+                    # Happens for example in `(_ for _ in []).send.__module__`.
+                    yield builtins
+                else:
+                    try:
+                        # TODO use sys.modules, __module__ can be faked.
+                        yield sys.modules[imp_plz]
+                    except KeyError:
+                        # __module__ can be something arbitrary that doesn't exist.
+                        yield builtins
+
+        return list(reversed(list(get())))
+
+    def execute_operation(self, other_access_handle, operator):
+        other_access = other_access_handle.access
+        op = _OPERATORS[operator]
+        return self._create_access_path(op(self._obj, other_access._obj))
+
+    def needs_type_completions(self):
+        return inspect.isclass(self._obj) and self._obj != type
+
+    def get_signature_params(self):
+        obj = self._obj
+        if py_version < 33:
+            raise ValueError("inspect.signature was introduced in 3.3")
+        if py_version == 34:
+            # In 3.4 inspect.signature are wrong for str and int. This has
+            # been fixed in 3.5. The signature of object is returned,
+            # because no signature was found for str. Here we imitate 3.5
+            # logic and just ignore the signature if the magic methods
+            # don't match object.
+            # 3.3 doesn't even have the logic and returns nothing for str
+            # and classes that inherit from object.
+            user_def = inspect._signature_get_user_defined_method
+            if (inspect.isclass(obj)
+                    and not user_def(type(obj), '__init__')
+                    and not user_def(type(obj), '__new__')
+                    and (obj.__init__ != object.__init__
+                         or obj.__new__ != object.__new__)):
+                raise ValueError
+
+        try:
+            signature = inspect.signature(obj)
+        except (RuntimeError, TypeError):
+            # Reading the code of the function in Python 3.6 implies there are
+            # at least these errors that might occur if something is wrong with
+            # the signature. In that case we just want a simple escape for now.
+            raise ValueError
+        return [
+            SignatureParam(
+                name=p.name,
+                has_default=p.default is not p.empty,
+                default=self._create_access_path(p.default),
+                has_annotation=p.annotation is not p.empty,
+                annotation=self._create_access_path(p.annotation),
+                kind_name=str(p.kind)
+            ) for p in signature.parameters.values()
+        ]
+
+    def negate(self):
+        return self._create_access_path(-self._obj)
+
+    def dict_values(self):
+        return [self._create_access_path(v) for v in self._obj.values()]
+
+    def is_super_class(self, exception):
+        return issubclass(exception, self._obj)
+
+    def get_dir_infos(self):
+        """
+        Used to return a couple of infos that are needed when accessing the sub
+        objects of an objects
+        """
+        # TODO is_allowed_getattr might raise an AttributeError
+        tuples = dict(
+            (force_unicode(name), self.is_allowed_getattr(name))
+            for name in self.dir()
+        )
+        return self.needs_type_completions(), tuples
+
+
+def _is_class_instance(obj):
+    """Like inspect.* methods."""
+    try:
+        cls = obj.__class__
+    except AttributeError:
+        return False
+    else:
+        return cls != type and not issubclass(cls, NOT_CLASS_TYPES)
+
+
+if py_version >= 35:
+    exec(compile(dedent("""
+        async def _coroutine(): pass
+        _coroutine = _coroutine()
+        CoroutineType = type(_coroutine)
+        _coroutine.close()  # Prevent ResourceWarning
+    """), 'blub', 'exec'))
+    _coroutine_wrapper = _coroutine.__await__()
+else:
+    _coroutine = None
+    _coroutine_wrapper = None
+
+if py_version >= 36:
+    exec(compile(dedent("""
+        async def _async_generator():
+            yield
+        _async_generator = _async_generator()
+        AsyncGeneratorType = type(_async_generator)
+    """), 'blub', 'exec'))
+else:
+    _async_generator = None
+
+class _SPECIAL_OBJECTS(object):
+    FUNCTION_CLASS = types.FunctionType
+    BOUND_METHOD_CLASS = type(DirectObjectAccess(None, None).py__bool__)
+    MODULE_CLASS = types.ModuleType
+    GENERATOR_OBJECT = _a_generator(1.0)
+    BUILTINS = builtins
+    COROUTINE = _coroutine
+    COROUTINE_WRAPPER = _coroutine_wrapper
+    ASYNC_GENERATOR = _async_generator
+
+
+def get_special_object(evaluator, identifier):
+    obj = getattr(_SPECIAL_OBJECTS, identifier)
+    return create_access_path(evaluator, obj)

jedi/evaluate/compiled/context.py

@@ -0,0 +1,483 @@
+"""
+Imitate the parser representation.
+"""
+import re
+from functools import partial
+
+from jedi import debug
+from jedi._compatibility import force_unicode, Parameter
+from jedi.cache import underscore_memoization, memoize_method
+from jedi.evaluate.filters import AbstractFilter, AbstractNameDefinition, \
+    ContextNameMixin
+from jedi.evaluate.base_context import Context, ContextSet
+from jedi.evaluate.lazy_context import LazyKnownContext
+from jedi.evaluate.compiled.access import _sentinel
+from jedi.evaluate.cache import evaluator_function_cache
+from jedi.evaluate.helpers import reraise_as_evaluator
+from . import fake
+
+
+class CheckAttribute(object):
+    """Raises an AttributeError if the attribute X isn't available."""
+    def __init__(self, func):
+        self.func = func
+        # Remove the py in front of e.g. py__call__.
+        self.check_name = force_unicode(func.__name__[2:])
+
+    def __get__(self, instance, owner):
+        if instance is None:
+            return self
+
+        # This might raise an AttributeError. That's wanted.
+        if self.check_name == '__iter__':
+            # Python iterators are a bit strange, because there's no need for
+            # the __iter__ function as long as __getitem__ is defined (it will
+            # just start with __getitem__(0). This is especially true for
+            # Python 2 strings, where `str.__iter__` is not even defined.
+            if not instance.access_handle.has_iter():
+                raise AttributeError
+        else:
+            instance.access_handle.getattr(self.check_name)
+        return partial(self.func, instance)
+
+
+class CompiledObject(Context):
+    def __init__(self, evaluator, access_handle, parent_context=None, faked_class=None):
+        super(CompiledObject, self).__init__(evaluator, parent_context)
+        self.access_handle = access_handle
+        # This attribute will not be set for most classes, except for fakes.
+        self.tree_node = faked_class
+
+    @CheckAttribute
+    def py__call__(self, params):
+        if self.tree_node is not None and self.tree_node.type == 'funcdef':
+            from jedi.evaluate.context.function import FunctionContext
+            return FunctionContext(
+                self.evaluator,
+                parent_context=self.parent_context,
+                tree_node=self.tree_node
+            ).py__call__(params)
+        if self.access_handle.is_class():
+            from jedi.evaluate.context import CompiledInstance
+            return ContextSet(CompiledInstance(self.evaluator, self.parent_context, self, params))
+        else:
+            return ContextSet.from_iterable(self._execute_function(params))
+
+    @CheckAttribute
+    def py__class__(self):
+        return create_from_access_path(self.evaluator, self.access_handle.py__class__())
+
+    @CheckAttribute
+    def py__mro__(self):
+        return (self,) + tuple(
+            create_from_access_path(self.evaluator, access)
+            for access in self.access_handle.py__mro__accesses()
+        )
+
+    @CheckAttribute
+    def py__bases__(self):
+        return tuple(
+            create_from_access_path(self.evaluator, access)
+            for access in self.access_handle.py__bases__()
+        )
+
+    @CheckAttribute
+    def py__path__(self):
+        return self.access_handle.py__path__()
+
+    def py__bool__(self):
+        return self.access_handle.py__bool__()
+
+    def py__file__(self):
+        return self.access_handle.py__file__()
+
+    def is_class(self):
+        return self.access_handle.is_class()
+
+    def py__doc__(self, include_call_signature=False):
+        return self.access_handle.py__doc__()
+
+    def get_param_names(self):
+        try:
+            signature_params = self.access_handle.get_signature_params()
+        except ValueError:  # Has no signature
+            params_str, ret = self._parse_function_doc()
+            tokens = params_str.split(',')
+            if self.access_handle.ismethoddescriptor():
+                tokens.insert(0, 'self')
+            for p in tokens:
+                parts = p.strip().split('=')
+                yield UnresolvableParamName(self, parts[0])
+        else:
+            for signature_param in signature_params:
+                yield SignatureParamName(self, signature_param)
+
+    def __repr__(self):
+        return '<%s: %s>' % (self.__class__.__name__, self.access_handle.get_repr())
+
+    @underscore_memoization
+    def _parse_function_doc(self):
+        doc = self.py__doc__()
+        if doc is None:
+            return '', ''
+
+        return _parse_function_doc(doc)
+
+    @property
+    def api_type(self):
+        return self.access_handle.get_api_type()
+
+    @underscore_memoization
+    def _cls(self):
+        """
+        We used to limit the lookups for instantiated objects like list(), but
+        this is not the case anymore. Python itself
+        """
+        # Ensures that a CompiledObject is returned that is not an instance (like list)
+        return self
+
+    def get_filters(self, search_global=False, is_instance=False,
+                    until_position=None, origin_scope=None):
+        yield self._ensure_one_filter(is_instance)
+
+    @memoize_method
+    def _ensure_one_filter(self, is_instance):
+        """
+        search_global shouldn't change the fact that there's one dict, this way
+        there's only one `object`.
+        """
+        return CompiledObjectFilter(self.evaluator, self, is_instance)
+
+    @CheckAttribute
+    def py__getitem__(self, index):
+        with reraise_as_evaluator(IndexError, KeyError, TypeError):
+            access = self.access_handle.py__getitem__(index)
+        if access is None:
+            return ContextSet()
+
+        return ContextSet(create_from_access_path(self.evaluator, access))
+
+    @CheckAttribute
+    def py__iter__(self):
+        for access in self.access_handle.py__iter__list():
+            yield LazyKnownContext(create_from_access_path(self.evaluator, access))
+
+    def py__name__(self):
+        return self.access_handle.py__name__()
+
+    @property
+    def name(self):
+        name = self.py__name__()
+        if name is None:
+            name = self.access_handle.get_repr()
+        return CompiledContextName(self, name)
+
+    def _execute_function(self, params):
+        from jedi.evaluate import docstrings
+        from jedi.evaluate.compiled import builtin_from_name
+        if self.api_type != 'function':
+            return
+
+        for name in self._parse_function_doc()[1].split():
+            try:
+                # TODO wtf is this? this is exactly the same as the thing
+                # below. It uses getattr as well.
+                self.evaluator.builtins_module.access_handle.getattr(name)
+            except AttributeError:
+                continue
+            else:
+                bltn_obj = builtin_from_name(self.evaluator, name)
+                for result in bltn_obj.execute(params):
+                    yield result
+        for type_ in docstrings.infer_return_types(self):
+            yield type_
+
+    def dict_values(self):
+        return ContextSet.from_iterable(
+            create_from_access_path(self.evaluator, access)
+            for access in self.access_handle.dict_values()
+        )
+
+    def get_safe_value(self, default=_sentinel):
+        try:
+            return self.access_handle.get_safe_value()
+        except ValueError:
+            if default == _sentinel:
+                raise
+            return default
+
+    def execute_operation(self, other, operator):
+        return create_from_access_path(
+            self.evaluator,
+            self.access_handle.execute_operation(other.access_handle, operator)
+        )
+
+    def negate(self):
+        return create_from_access_path(self.evaluator, self.access_handle.negate())
+
+    def is_super_class(self, exception):
+        return self.access_handle.is_super_class(exception)
+
+
+class CompiledName(AbstractNameDefinition):
+    def __init__(self, evaluator, parent_context, name):
+        self._evaluator = evaluator
+        self.parent_context = parent_context
+        self.string_name = name
+
+    def __repr__(self):
+        try:
+            name = self.parent_context.name  # __name__ is not defined all the time
+        except AttributeError:
+            name = None
+        return '<%s: (%s).%s>' % (self.__class__.__name__, name, self.string_name)
+
+    @property
+    def api_type(self):
+        return next(iter(self.infer())).api_type
+
+    @underscore_memoization
+    def infer(self):
+        return ContextSet(create_from_name(
+            self._evaluator, self.parent_context, self.string_name
+        ))
+
+
+class SignatureParamName(AbstractNameDefinition):
+    api_type = u'param'
+
+    def __init__(self, compiled_obj, signature_param):
+        self.parent_context = compiled_obj.parent_context
+        self._signature_param = signature_param
+
+    @property
+    def string_name(self):
+        return self._signature_param.name
+
+    def get_kind(self):
+        return getattr(Parameter, self._signature_param.kind_name)
+
+    def is_keyword_param(self):
+        return self._signature_param
+
+    def infer(self):
+        p = self._signature_param
+        evaluator = self.parent_context.evaluator
+        contexts = ContextSet()
+        if p.has_default:
+            contexts = ContextSet(create_from_access_path(evaluator, p.default))
+        if p.has_annotation:
+            annotation = create_from_access_path(evaluator, p.annotation)
+            contexts |= annotation.execute_evaluated()
+        return contexts
+
+
+class UnresolvableParamName(AbstractNameDefinition):
+    api_type = u'param'
+
+    def __init__(self, compiled_obj, name):
+        self.parent_context = compiled_obj.parent_context
+        self.string_name = name
+
+    def get_kind(self):
+        return Parameter.POSITIONAL_ONLY
+
+    def infer(self):
+        return ContextSet()
+
+
+class CompiledContextName(ContextNameMixin, AbstractNameDefinition):
+    def __init__(self, context, name):
+        self.string_name = name
+        self._context = context
+        self.parent_context = context.parent_context
+
+
+class EmptyCompiledName(AbstractNameDefinition):
+    """
+    Accessing some names will raise an exception. To avoid not having any
+    completions, just give Jedi the option to return this object. It infers to
+    nothing.
+    """
+    def __init__(self, evaluator, name):
+        self.parent_context = evaluator.builtins_module
+        self.string_name = name
+
+    def infer(self):
+        return ContextSet()
+
+
+class CompiledObjectFilter(AbstractFilter):
+    name_class = CompiledName
+
+    def __init__(self, evaluator, compiled_object, is_instance=False):
+        self._evaluator = evaluator
+        self._compiled_object = compiled_object
+        self._is_instance = is_instance
+
+    def get(self, name):
+        return self._get(
+            name,
+            lambda: self._compiled_object.access_handle.is_allowed_getattr(name),
+            lambda: self._compiled_object.access_handle.dir(),
+            check_has_attribute=True
+        )
+
+    def _get(self, name, allowed_getattr_callback, dir_callback, check_has_attribute=False):
+        """
+        To remove quite a few access calls we introduced the callback here.
+        """
+        has_attribute, is_descriptor = allowed_getattr_callback()
+        if check_has_attribute and not has_attribute:
+            return []
+
+        # Always use unicode objects in Python 2 from here.
+        name = force_unicode(name)
+
+        if is_descriptor or not has_attribute:
+            return [self._get_cached_name(name, is_empty=True)]
+
+        if self._is_instance and name not in dir_callback():
+            return []
+        return [self._get_cached_name(name)]
+
+    @memoize_method
+    def _get_cached_name(self, name, is_empty=False):
+        if is_empty:
+            return EmptyCompiledName(self._evaluator, name)
+        else:
+            return self._create_name(name)
+
+    def values(self):
+        from jedi.evaluate.compiled import builtin_from_name
+        names = []
+        needs_type_completions, dir_infos = self._compiled_object.access_handle.get_dir_infos()
+        for name in dir_infos:
+            names += self._get(
+                name,
+                lambda: dir_infos[name],
+                lambda: dir_infos.keys(),
+            )
+
+        # ``dir`` doesn't include the type names.
+        if not self._is_instance and needs_type_completions:
+            for filter in builtin_from_name(self._evaluator, u'type').get_filters():
+                names += filter.values()
+        return names
+
+    def _create_name(self, name):
+        return self.name_class(self._evaluator, self._compiled_object, name)
+
+
+docstr_defaults = {
+    'floating point number': u'float',
+    'character': u'str',
+    'integer': u'int',
+    'dictionary': u'dict',
+    'string': u'str',
+}
+
+
+def _parse_function_doc(doc):
+    """
+    Takes a function and returns the params and return value as a tuple.
+    This is nothing more than a docstring parser.
+
+    TODO docstrings like utime(path, (atime, mtime)) and a(b [, b]) -> None
+    TODO docstrings like 'tuple of integers'
+    """
+    doc = force_unicode(doc)
+    # parse round parentheses: def func(a, (b,c))
+    try:
+        count = 0
+        start = doc.index('(')
+        for i, s in enumerate(doc[start:]):
+            if s == '(':
+                count += 1
+            elif s == ')':
+                count -= 1
+            if count == 0:
+                end = start + i
+                break
+        param_str = doc[start + 1:end]
+    except (ValueError, UnboundLocalError):
+        # ValueError for doc.index
+        # UnboundLocalError for undefined end in last line
+        debug.dbg('no brackets found - no param')
+        end = 0
+        param_str = u''
+    else:
+        # remove square brackets, that show an optional param ( = None)
+        def change_options(m):
+            args = m.group(1).split(',')
+            for i, a in enumerate(args):
+                if a and '=' not in a:
+                    args[i] += '=None'
+            return ','.join(args)
+
+        while True:
+            param_str, changes = re.subn(r' ?\[([^\[\]]+)\]',
+                                         change_options, param_str)
+            if changes == 0:
+                break
+    param_str = param_str.replace('-', '_')  # see: isinstance.__doc__
+
+    # parse return value
+    r = re.search(u'-[>-]* ', doc[end:end + 7])
+    if r is None:
+        ret = u''
+    else:
+        index = end + r.end()
+        # get result type, which can contain newlines
+        pattern = re.compile(r'(,\n|[^\n-])+')
+        ret_str = pattern.match(doc, index).group(0).strip()
+        # New object -> object()
+        ret_str = re.sub(r'[nN]ew (.*)', r'\1()', ret_str)
+
+        ret = docstr_defaults.get(ret_str, ret_str)
+
+    return param_str, ret
+
+
+def create_from_name(evaluator, compiled_object, name):
+    faked = None
+    try:
+        faked = fake.get_faked_with_parent_context(compiled_object, name)
+    except fake.FakeDoesNotExist:
+        pass
+
+    access = compiled_object.access_handle.getattr(name, default=None)
+    parent_context = compiled_object
+    if parent_context.is_class():
+        parent_context = parent_context.parent_context
+    return create_cached_compiled_object(
+        evaluator, access, parent_context=parent_context, faked=faked
+    )
+
+
+def _normalize_create_args(func):
+    """The cache doesn't care about keyword vs. normal args."""
+    def wrapper(evaluator, obj, parent_context=None, faked=None):
+        return func(evaluator, obj, parent_context, faked)
+    return wrapper
+
+
+def create_from_access_path(evaluator, access_path):
+    parent_context = None
+    for name, access in access_path.accesses:
+        try:
+            if parent_context is None:
+                faked = fake.get_faked_module(evaluator, access_path.accesses[0][0])
+            else:
+                faked = fake.get_faked_with_parent_context(parent_context, name)
+        except fake.FakeDoesNotExist:
+            faked = None
+
+        parent_context = create_cached_compiled_object(evaluator, access, parent_context, faked)
+    return parent_context
+
+
+@_normalize_create_args
+@evaluator_function_cache()
+def create_cached_compiled_object(evaluator, access_handle, parent_context, faked):
+    return CompiledObject(evaluator, access_handle, parent_context, faked)

jedi/evaluate/compiled/fake.py

@@ -5,73 +5,59 @@ mixing in Python code, the autocompletion should work much better for builtins.
 """
 
 import os
-import inspect
-import types
 from itertools import chain
 
-from parso.python import tree
+from jedi._compatibility import unicode
 
-from jedi._compatibility import is_py3, builtins, unicode, is_py34
-
-modules = {}
+fake_modules = {}
 
 
-MethodDescriptorType = type(str.replace)
-# These are not considered classes and access is granted even though they have
-# a __class__ attribute.
-NOT_CLASS_TYPES = (
-    types.BuiltinFunctionType,
-    types.CodeType,
-    types.FrameType,
-    types.FunctionType,
-    types.GeneratorType,
-    types.GetSetDescriptorType,
-    types.LambdaType,
-    types.MemberDescriptorType,
-    types.MethodType,
-    types.ModuleType,
-    types.TracebackType,
-    MethodDescriptorType
-)
+def _get_path_dict():
+    path = os.path.dirname(os.path.abspath(__file__))
+    base_path = os.path.join(path, 'fake')
+    dct = {}
+    for file_name in os.listdir(base_path):
+        if file_name.endswith('.pym'):
+            dct[file_name[:-4]] = os.path.join(base_path, file_name)
+    return dct
 
-if is_py3:
-    NOT_CLASS_TYPES += (
-        types.MappingProxyType,
-        types.SimpleNamespace
-    )
-    if is_py34:
-        NOT_CLASS_TYPES += (types.DynamicClassAttribute,)
+
+_path_dict = _get_path_dict()
 
 
 class FakeDoesNotExist(Exception):
     pass
 
 
-def _load_faked_module(grammar, module):
-    module_name = module.__name__
-    if module_name == '__builtin__' and not is_py3:
-        module_name = 'builtins'
+def _load_faked_module(evaluator, module_name):
+    try:
+        return fake_modules[module_name]
+    except KeyError:
+        pass
+
+    check_module_name = module_name
+    if module_name == '__builtin__' and evaluator.environment.version_info.major == 2:
+        check_module_name = 'builtins'
 
     try:
-        return modules[module_name]
+        path = _path_dict[check_module_name]
     except KeyError:
-        path = os.path.dirname(os.path.abspath(__file__))
-        try:
-            with open(os.path.join(path, 'fake', module_name) + '.pym') as f:
-                source = f.read()
-        except IOError:
-            modules[module_name] = None
-            return
-        modules[module_name] = m = grammar.parse(unicode(source))
+        fake_modules[module_name] = None
+        return
 
-        if module_name == 'builtins' and not is_py3:
-            # There are two implementations of `open` for either python 2/3.
-            # -> Rename the python2 version (`look at fake/builtins.pym`).
-            open_func = _search_scope(m, 'open')
-            open_func.children[1].value = 'open_python3'
-            open_func = _search_scope(m, 'open_python2')
-            open_func.children[1].value = 'open'
-        return m
+    with open(path) as f:
+        source = f.read()
+
+    fake_modules[module_name] = m = evaluator.latest_grammar.parse(unicode(source))
+
+    if check_module_name != module_name:
+        # There are two implementations of `open` for either python 2/3.
+        # -> Rename the python2 version (`look at fake/builtins.pym`).
+        open_func = _search_scope(m, 'open')
+        open_func.children[1].value = 'open_python3'
+        open_func = _search_scope(m, 'open_python2')
+        open_func.children[1].value = 'open'
+    return m
 
 
 def _search_scope(scope, obj_name):
@@ -80,134 +66,17 @@ def _search_scope(scope, obj_name):
             return s
 
 
-def get_module(obj):
-    if inspect.ismodule(obj):
-        return obj
-    try:
-        obj = obj.__objclass__
-    except AttributeError:
-        pass
-
-    try:
-        imp_plz = obj.__module__
-    except AttributeError:
-        # Unfortunately in some cases like `int` there's no __module__
-        return builtins
-    else:
-        if imp_plz is None:
-            # Happens for example in `(_ for _ in []).send.__module__`.
-            return builtins
-        else:
-            try:
-                return __import__(imp_plz)
-            except ImportError:
-                # __module__ can be something arbitrary that doesn't exist.
-                return builtins
-
-
-def _faked(grammar, module, obj, name):
-    # Crazy underscore actions to try to escape all the internal madness.
-    if module is None:
-        module = get_module(obj)
-
-    faked_mod = _load_faked_module(grammar, module)
-    if faked_mod is None:
-        return None, None
-
-    # Having the module as a `parser.python.tree.Module`, we need to scan
-    # for methods.
-    if name is None:
-        if inspect.isbuiltin(obj) or inspect.isclass(obj):
-            return _search_scope(faked_mod, obj.__name__), faked_mod
-        elif not inspect.isclass(obj):
-            # object is a method or descriptor
-            try:
-                objclass = obj.__objclass__
-            except AttributeError:
-                return None, None
-            else:
-                cls = _search_scope(faked_mod, objclass.__name__)
-                if cls is None:
-                    return None, None
-                return _search_scope(cls, obj.__name__), faked_mod
-    else:
-        if obj is module:
-            return _search_scope(faked_mod, name), faked_mod
-        else:
-            try:
-                cls_name = obj.__name__
-            except AttributeError:
-                return None, None
-            cls = _search_scope(faked_mod, cls_name)
-            if cls is None:
-                return None, None
-            return _search_scope(cls, name), faked_mod
-    return None, None
-
-
-def memoize_faked(obj):
-    """
-    A typical memoize function that ignores issues with non hashable results.
-    """
-    cache = obj.cache = {}
-
-    def memoizer(*args, **kwargs):
-        key = (obj, args, frozenset(kwargs.items()))
-        try:
-            result = cache[key]
-        except (TypeError, ValueError):
-            return obj(*args, **kwargs)
-        except KeyError:
-            result = obj(*args, **kwargs)
-            if result is not None:
-                cache[key] = obj(*args, **kwargs)
-            return result
-        else:
-            return result
-    return memoizer
-
-
-@memoize_faked
-def _get_faked(grammar, module, obj, name=None):
-    result, fake_module = _faked(grammar, module, obj, name)
-    if result is None:
-        # We're not interested in classes. What we want is functions.
-        raise FakeDoesNotExist
-    elif result.type == 'classdef':
-        return result, fake_module
-    else:
-        # Set the docstr which was previously not set (faked modules don't
-        # contain it).
-        assert result.type == 'funcdef'
-        doc = '"""%s"""' % obj.__doc__  # TODO need escapes.
-        suite = result.children[-1]
-        string = tree.String(doc, (0, 0), '')
-        new_line = tree.Newline('\n', (0, 0))
-        docstr_node = tree.PythonNode('simple_stmt', [string, new_line])
-        suite.children.insert(1, docstr_node)
-        return result, fake_module
-
-
-def get_faked(evaluator, module, obj, name=None, parent_context=None):
-    if parent_context and parent_context.tree_node is not None:
+def get_faked_with_parent_context(parent_context, name):
+    if parent_context.tree_node is not None:
         # Try to search in already clearly defined stuff.
         found = _search_scope(parent_context.tree_node, name)
         if found is not None:
             return found
-        else:
-            raise FakeDoesNotExist
-
-    faked, fake_module = _get_faked(evaluator.latest_grammar, module and module.obj, obj, name)
-    if module is not None:
-        module.get_used_names = fake_module.get_used_names
-    return faked
+    raise FakeDoesNotExist
 
 
-def is_class_instance(obj):
-    """Like inspect.* methods."""
-    try:
-        cls = obj.__class__
-    except AttributeError:
-        return False
-    else:
-        return cls != type and not issubclass(cls, NOT_CLASS_TYPES)
+def get_faked_module(evaluator, string_name):
+    module = _load_faked_module(evaluator, string_name)
+    if module is None:
+        raise FakeDoesNotExist
+    return module

jedi/evaluate/compiled/fake/builtins.pym

@@ -201,10 +201,13 @@ class dict():
         # has a strange docstr
         pass
 
+    def __getitem__(self, obj):
+        return self.__elements[obj]
+
     def get(self, k, d=None):
         # TODO implement
         try:
-            #return self.__elements[k]
+            return self.__elements[k]
             pass
         except KeyError:
             return d

jedi/evaluate/compiled/getattr_static.py

@@ -10,6 +10,7 @@ from jedi._compatibility import py_version
 
 _sentinel = object()
 
+
 def _check_instance(obj, attr):
     instance_dict = {}
     try:
@@ -28,6 +29,7 @@ def _check_class(klass, attr):
                 pass
     return _sentinel
 
+
 def _is_type(obj):
     try:
         _static_getmro(obj)
@@ -87,7 +89,7 @@ else:
             return getattr(klass, '__dict__', _sentinel)
         return _shadowed_dict_newstyle(klass)
 
-    class _OldStyleClass():
+    class _OldStyleClass:
         pass
 
     _oldstyle_instance_type = type(_OldStyleClass())
@@ -122,7 +124,7 @@ def _safe_hasattr(obj, name):
 
 
 def _safe_is_data_descriptor(obj):
-    return (_safe_hasattr(obj, '__set__') or _safe_hasattr(obj, '__delete__'))
+    return _safe_hasattr(obj, '__set__') or _safe_hasattr(obj, '__delete__')
 
 
 def getattr_static(obj, attr, default=_sentinel):
@@ -143,8 +145,7 @@ def getattr_static(obj, attr, default=_sentinel):
     if not _is_type(obj):
         klass = _get_type(obj)
         dict_attr = _shadowed_dict(klass)
-        if (dict_attr is _sentinel or
-            type(dict_attr) is types.MemberDescriptorType):
+        if (dict_attr is _sentinel or type(dict_attr) is types.MemberDescriptorType):
             instance_result = _check_instance(obj, attr)
     else:
         klass = obj

jedi/evaluate/compiled/mixed.py

@@ -5,13 +5,18 @@ Used only for REPL Completion.
 import inspect
 import os
 
+from jedi.parser_utils import get_cached_code_lines
+
 from jedi import settings
 from jedi.evaluate import compiled
 from jedi.cache import underscore_memoization
 from jedi.evaluate import imports
-from jedi.evaluate.context import Context
+from jedi.evaluate.base_context import Context, ContextSet
+from jedi.evaluate.context import ModuleContext
 from jedi.evaluate.cache import evaluator_function_cache
 from jedi.evaluate.compiled.getattr_static import getattr_static
+from jedi.evaluate.compiled.access import compiled_objects_cache
+from jedi.evaluate.compiled.context import create_cached_compiled_object
 
 
 class MixedObject(object):
@@ -36,14 +41,11 @@ class MixedObject(object):
         self.parent_context = parent_context
         self.compiled_object = compiled_object
         self._context = tree_context
-        self.obj = compiled_object.obj
+        self.access_handle = compiled_object.access_handle
 
     # We have to overwrite everything that has to do with trailers, name
     # lookups and filters to make it possible to route name lookups towards
     # compiled objects and the rest towards tree node contexts.
-    def eval_trailer(*args, **kwags):
-        return Context.eval_trailer(*args, **kwags)
-
     def py__getattribute__(*args, **kwargs):
         return Context.py__getattribute__(*args, **kwargs)
 
@@ -51,7 +53,7 @@ class MixedObject(object):
         yield MixedObjectFilter(self.evaluator, self)
 
     def __repr__(self):
-        return '<%s: %s>' % (type(self).__name__, repr(self.obj))
+        return '<%s: %s>' % (type(self).__name__, self.access_handle.get_repr())
 
     def __getattr__(self, name):
         return getattr(self._context, name)
@@ -66,7 +68,7 @@ class MixedName(compiled.CompiledName):
         contexts = list(self.infer())
         if not contexts:
             # This means a start_pos that doesn't exist (compiled objects).
-            return (0, 0)
+            return 0, 0
         return contexts[0].name.start_pos
 
     @start_pos.setter
@@ -76,16 +78,12 @@ class MixedName(compiled.CompiledName):
 
     @underscore_memoization
     def infer(self):
-        obj = self.parent_context.obj
-        try:
-            # TODO use logic from compiled.CompiledObjectFilter
-            obj = getattr(obj, self.string_name)
-        except AttributeError:
-            # Happens e.g. in properties of
-            # PyQt4.QtGui.QStyleOptionComboBox.currentText
-            # -> just set it to None
-            obj = None
-        return [_create(self._evaluator, obj, parent_context=self.parent_context)]
+        access_handle = self.parent_context.access_handle
+        # TODO use logic from compiled.CompiledObjectFilter
+        access_handle = access_handle.getattr(self.string_name, default=None)
+        return ContextSet(
+            _create(self._evaluator, access_handle, parent_context=self.parent_context)
+        )
 
     @property
     def api_type(self):
@@ -105,17 +103,17 @@ class MixedObjectFilter(compiled.CompiledObjectFilter):
 
 
 @evaluator_function_cache()
-def _load_module(evaluator, path, python_object):
-    module = evaluator.grammar.parse(
+def _load_module(evaluator, path):
+    module_node = evaluator.grammar.parse(
         path=path,
         cache=True,
-        diff_cache=True,
+        diff_cache=settings.fast_parser,
         cache_path=settings.cache_directory
     ).get_root_node()
-    python_module = inspect.getmodule(python_object)
-
-    evaluator.modules[python_module.__name__] = module
-    return module
+    # python_module = inspect.getmodule(python_object)
+    # TODO we should actually make something like this possible.
+    #evaluator.modules[python_module.__name__] = module_node
+    return module_node
 
 
 def _get_object_to_check(python_object):
@@ -135,40 +133,43 @@ def _get_object_to_check(python_object):
         raise TypeError  # Prevents computation of `repr` within inspect.
 
 
-def find_syntax_node_name(evaluator, python_object):
+def _find_syntax_node_name(evaluator, access_handle):
+    # TODO accessing this is bad, but it probably doesn't matter that much,
+    # because we're working with interpreteters only here.
+    python_object = access_handle.access._obj
     try:
         python_object = _get_object_to_check(python_object)
         path = inspect.getsourcefile(python_object)
     except TypeError:
         # The type might not be known (e.g. class_with_dict.__weakref__)
-        return None, None
+        return None
     if path is None or not os.path.exists(path):
         # The path might not exist or be e.g. <stdin>.
-        return None, None
+        return None
 
-    module = _load_module(evaluator, path, python_object)
+    module_node = _load_module(evaluator, path)
 
     if inspect.ismodule(python_object):
         # We don't need to check names for modules, because there's not really
         # a way to write a module in a module in Python (and also __name__ can
         # be something like ``email.utils``).
-        return module, path
+        code_lines = get_cached_code_lines(evaluator.grammar, path)
+        return module_node, module_node, path, code_lines
 
     try:
         name_str = python_object.__name__
     except AttributeError:
         # Stuff like python_function.__code__.
-        return None, None
+        return None
 
     if name_str == '<lambda>':
-        return None, None  # It's too hard to find lambdas.
+        return None  # It's too hard to find lambdas.
 
     # Doesn't always work (e.g. os.stat_result)
-    try:
-        names = module.get_used_names()[name_str]
-    except KeyError:
-        return None, None
+    names = module_node.get_used_names().get(name_str, [])
     names = [n for n in names if n.is_definition()]
+    if not names:
+        return None
 
     try:
         code = python_object.__code__
@@ -184,34 +185,40 @@ def find_syntax_node_name(evaluator, python_object):
         # There's a chance that the object is not available anymore, because
         # the code has changed in the background.
         if line_names:
-            return line_names[-1].parent, path
+            names = line_names
 
+    code_lines = get_cached_code_lines(evaluator.grammar, path)
     # It's really hard to actually get the right definition, here as a last
     # resort we just return the last one. This chance might lead to odd
     # completions at some points but will lead to mostly correct type
     # inference, because people tend to define a public name in a module only
     # once.
-    return names[-1].parent, path
+    return module_node, names[-1].parent, path, code_lines
 
 
-@compiled.compiled_objects_cache('mixed_cache')
-def _create(evaluator, obj, parent_context=None, *args):
-    tree_node, path = find_syntax_node_name(evaluator, obj)
+@compiled_objects_cache('mixed_cache')
+def _create(evaluator, access_handle, parent_context, *args):
+    compiled_object = create_cached_compiled_object(
+        evaluator, access_handle, parent_context=parent_context.compiled_object)
 
-    compiled_object = compiled.create(
-        evaluator, obj, parent_context=parent_context.compiled_object)
-    if tree_node is None:
+    result = _find_syntax_node_name(evaluator, access_handle)
+    if result is None:
         return compiled_object
 
-    module_node = tree_node.get_root_node()
+    module_node, tree_node, path, code_lines = result
+
     if parent_context.tree_node.get_root_node() == module_node:
         module_context = parent_context.get_root_context()
     else:
-        from jedi.evaluate.representation import ModuleContext
-        module_context = ModuleContext(evaluator, module_node, path=path)
+        module_context = ModuleContext(
+            evaluator, module_node,
+            path=path,
+            code_lines=code_lines,
+        )
         # TODO this __name__ is probably wrong.
         name = compiled_object.get_root_context().py__name__()
-        imports.add_module(evaluator, name, module_context)
+        if name is not None:
+            imports.add_module_to_cache(evaluator, name, module_context)
 
     tree_context = module_context.create_context(
         tree_node,
@@ -219,7 +226,7 @@ def _create(evaluator, obj, parent_context=None, *args):
         node_is_object=True
     )
     if tree_node.type == 'classdef':
-        if not inspect.isclass(obj):
+        if not access_handle.is_class():
             # Is an instance, not a class.
             tree_context, = tree_context.execute_evaluated()
 
@@ -229,4 +236,3 @@ def _create(evaluator, obj, parent_context=None, *args):
         compiled_object,
         tree_context=tree_context
     )
-

jedi/evaluate/compiled/subprocess/__init__.py

@@ -0,0 +1,397 @@
+"""
+Makes it possible to do the compiled analysis in a subprocess. This has two
+goals:
+
+1. Making it safer - Segfaults and RuntimeErrors as well as stdout/stderr can
+   be ignored and dealt with.
+2. Make it possible to handle different Python versions as well as virtualenvs.
+"""
+
+import os
+import sys
+import subprocess
+import socket
+import errno
+import weakref
+import traceback
+from functools import partial
+from threading import Thread
+try:
+    from queue import Queue, Empty
+except ImportError:
+    from Queue import Queue, Empty  # python 2.7
+
+from jedi._compatibility import queue, is_py3, force_unicode, \
+    pickle_dump, pickle_load, GeneralizedPopen, print_to_stderr
+from jedi import debug
+from jedi.cache import memoize_method
+from jedi.evaluate.compiled.subprocess import functions
+from jedi.evaluate.compiled.access import DirectObjectAccess, AccessPath, \
+    SignatureParam
+from jedi.api.exceptions import InternalError
+
+
+_MAIN_PATH = os.path.join(os.path.dirname(__file__), '__main__.py')
+
+
+def _enqueue_output(out, queue):
+    for line in iter(out.readline, b''):
+        queue.put(line)
+    out.close()
+
+
+def _add_stderr_to_debug(stderr_queue):
+    while True:
+        # Try to do some error reporting from the subprocess and print its
+        # stderr contents.
+        try:
+            line = stderr_queue.get_nowait()
+            line = line.decode('utf-8', 'replace')
+            debug.warning('stderr output: %s' % line.rstrip('\n'))
+        except Empty:
+            break
+
+
+def _get_function(name):
+    return getattr(functions, name)
+
+
+class _EvaluatorProcess(object):
+    def __init__(self, evaluator):
+        self._evaluator_weakref = weakref.ref(evaluator)
+        self._evaluator_id = id(evaluator)
+        self._handles = {}
+
+    def get_or_create_access_handle(self, obj):
+        id_ = id(obj)
+        try:
+            return self.get_access_handle(id_)
+        except KeyError:
+            access = DirectObjectAccess(self._evaluator_weakref(), obj)
+            handle = AccessHandle(self, access, id_)
+            self.set_access_handle(handle)
+            return handle
+
+    def get_access_handle(self, id_):
+        return self._handles[id_]
+
+    def set_access_handle(self, handle):
+        self._handles[handle.id] = handle
+
+
+class EvaluatorSameProcess(_EvaluatorProcess):
+    """
+    Basically just an easy access to functions.py. It has the same API
+    as EvaluatorSubprocess and does the same thing without using a subprocess.
+    This is necessary for the Interpreter process.
+    """
+    def __getattr__(self, name):
+        return partial(_get_function(name), self._evaluator_weakref())
+
+
+class EvaluatorSubprocess(_EvaluatorProcess):
+    def __init__(self, evaluator, compiled_subprocess):
+        super(EvaluatorSubprocess, self).__init__(evaluator)
+        self._used = False
+        self._compiled_subprocess = compiled_subprocess
+
+    def __getattr__(self, name):
+        func = _get_function(name)
+
+        def wrapper(*args, **kwargs):
+            self._used = True
+
+            result = self._compiled_subprocess.run(
+                self._evaluator_weakref(),
+                func,
+                args=args,
+                kwargs=kwargs,
+            )
+            # IMO it should be possible to create a hook in pickle.load to
+            # mess with the loaded objects. However it's extremely complicated
+            # to work around this so just do it with this call. ~ dave
+            return self._convert_access_handles(result)
+
+        return wrapper
+
+    def _convert_access_handles(self, obj):
+        if isinstance(obj, SignatureParam):
+            return SignatureParam(*self._convert_access_handles(tuple(obj)))
+        elif isinstance(obj, tuple):
+            return tuple(self._convert_access_handles(o) for o in obj)
+        elif isinstance(obj, list):
+            return [self._convert_access_handles(o) for o in obj]
+        elif isinstance(obj, AccessHandle):
+            try:
+                # Rewrite the access handle to one we're already having.
+                obj = self.get_access_handle(obj.id)
+            except KeyError:
+                obj.add_subprocess(self)
+                self.set_access_handle(obj)
+        elif isinstance(obj, AccessPath):
+            return AccessPath(self._convert_access_handles(obj.accesses))
+        return obj
+
+    def __del__(self):
+        if self._used and not self._compiled_subprocess.is_crashed:
+            self._compiled_subprocess.delete_evaluator(self._evaluator_id)
+
+
+class CompiledSubprocess(object):
+    is_crashed = False
+    # Start with 2, gets set after _get_info.
+    _pickle_protocol = 2
+
+    def __init__(self, executable):
+        self._executable = executable
+        self._evaluator_deletion_queue = queue.deque()
+
+    def __repr__(self):
+        pid = os.getpid()
+        return '<%s _executable=%r, _pickle_protocol=%r, is_crashed=%r, pid=%r>' % (
+            self.__class__.__name__,
+            self._executable,
+            self._pickle_protocol,
+            self.is_crashed,
+            pid,
+        )
+
+    @property
+    @memoize_method
+    def _process(self):
+        debug.dbg('Start environment subprocess %s', self._executable)
+        parso_path = sys.modules['parso'].__file__
+        args = (
+            self._executable,
+            _MAIN_PATH,
+            os.path.dirname(os.path.dirname(parso_path)),
+            '.'.join(str(x) for x in sys.version_info[:3]),
+        )
+        process = GeneralizedPopen(
+            args,
+            stdin=subprocess.PIPE,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            # Use system default buffering on Python 2 to improve performance
+            # (this is already the case on Python 3).
+            bufsize=-1
+        )
+        self._stderr_queue = Queue()
+        self._stderr_thread = t = Thread(
+            target=_enqueue_output,
+            args=(process.stderr, self._stderr_queue)
+        )
+        t.daemon = True
+        t.start()
+        return process
+
+    def run(self, evaluator, function, args=(), kwargs={}):
+        # Delete old evaluators.
+        while True:
+            try:
+                evaluator_id = self._evaluator_deletion_queue.pop()
+            except IndexError:
+                break
+            else:
+                self._send(evaluator_id, None)
+
+        assert callable(function)
+        return self._send(id(evaluator), function, args, kwargs)
+
+    def get_sys_path(self):
+        return self._send(None, functions.get_sys_path, (), {})
+
+    def _kill(self):
+        self.is_crashed = True
+        try:
+            self._process.kill()
+            self._process.wait()
+        except (AttributeError, TypeError):
+            # If the Python process is terminating, it will remove some modules
+            # earlier than others and in general it's unclear how to deal with
+            # that so we just ignore the exceptions here.
+            pass
+
+    def __del__(self):
+        if not self.is_crashed:
+            self._kill()
+
+    def _send(self, evaluator_id, function, args=(), kwargs={}):
+        if self.is_crashed:
+            raise InternalError("The subprocess %s has crashed." % self._executable)
+
+        if not is_py3:
+            # Python 2 compatibility
+            kwargs = {force_unicode(key): value for key, value in kwargs.items()}
+
+        data = evaluator_id, function, args, kwargs
+        try:
+            pickle_dump(data, self._process.stdin, self._pickle_protocol)
+        except (socket.error, IOError) as e:
+            # Once Python2 will be removed we can just use `BrokenPipeError`.
+            # Also, somehow in windows it returns EINVAL instead of EPIPE if
+            # the subprocess dies.
+            if e.errno not in (errno.EPIPE, errno.EINVAL):
+                # Not a broken pipe
+                raise
+            self._kill()
+            raise InternalError("The subprocess %s was killed. Maybe out of memory?"
+                                % self._executable)
+
+        try:
+            is_exception, traceback, result = pickle_load(self._process.stdout)
+        except EOFError as eof_error:
+            try:
+                stderr = self._process.stderr.read().decode('utf-8', 'replace')
+            except Exception as exc:
+                stderr = '<empty/not available (%r)>' % exc
+            self._kill()
+            _add_stderr_to_debug(self._stderr_queue)
+            raise InternalError(
+                "The subprocess %s has crashed (%r, stderr=%s)." % (
+                    self._executable,
+                    eof_error,
+                    stderr,
+                ))
+
+        _add_stderr_to_debug(self._stderr_queue)
+
+        if is_exception:
+            # Replace the attribute error message with a the traceback. It's
+            # way more informative.
+            result.args = (traceback,)
+            raise result
+        return result
+
+    def delete_evaluator(self, evaluator_id):
+        """
+        Currently we are not deleting evalutors instantly. They only get
+        deleted once the subprocess is used again. It would probably a better
+        solution to move all of this into a thread. However, the memory usage
+        of a single evaluator shouldn't be that high.
+        """
+        # With an argument - the evaluator gets deleted.
+        self._evaluator_deletion_queue.append(evaluator_id)
+
+
+class Listener(object):
+    def __init__(self, pickle_protocol):
+        self._evaluators = {}
+        # TODO refactor so we don't need to process anymore just handle
+        # controlling.
+        self._process = _EvaluatorProcess(Listener)
+        self._pickle_protocol = pickle_protocol
+
+    def _get_evaluator(self, function, evaluator_id):
+        from jedi.evaluate import Evaluator
+
+        try:
+            evaluator = self._evaluators[evaluator_id]
+        except KeyError:
+            from jedi.api.environment import InterpreterEnvironment
+            evaluator = Evaluator(
+                # The project is not actually needed. Nothing should need to
+                # access it.
+                project=None,
+                environment=InterpreterEnvironment()
+            )
+            self._evaluators[evaluator_id] = evaluator
+        return evaluator
+
+    def _run(self, evaluator_id, function, args, kwargs):
+        if evaluator_id is None:
+            return function(*args, **kwargs)
+        elif function is None:
+            del self._evaluators[evaluator_id]
+        else:
+            evaluator = self._get_evaluator(function, evaluator_id)
+
+            # Exchange all handles
+            args = list(args)
+            for i, arg in enumerate(args):
+                if isinstance(arg, AccessHandle):
+                    args[i] = evaluator.compiled_subprocess.get_access_handle(arg.id)
+            for key, value in kwargs.items():
+                if isinstance(value, AccessHandle):
+                    kwargs[key] = evaluator.compiled_subprocess.get_access_handle(value.id)
+
+            return function(evaluator, *args, **kwargs)
+
+    def listen(self):
+        stdout = sys.stdout
+        # Mute stdout. Nobody should actually be able to write to it,
+        # because stdout is used for IPC.
+        sys.stdout = open(os.devnull, 'w')
+        stdin = sys.stdin
+        if sys.version_info[0] > 2:
+            stdout = stdout.buffer
+            stdin = stdin.buffer
+        # Python 2 opens streams in text mode on Windows. Set stdout and stdin
+        # to binary mode.
+        elif sys.platform == 'win32':
+            import msvcrt
+            msvcrt.setmode(stdout.fileno(), os.O_BINARY)
+            msvcrt.setmode(stdin.fileno(), os.O_BINARY)
+
+        while True:
+            try:
+                payload = pickle_load(stdin)
+            except EOFError:
+                # It looks like the parent process closed.
+                # Don't make a big fuss here and just exit.
+                exit(0)
+            try:
+                result = False, None, self._run(*payload)
+            except Exception as e:
+                result = True, traceback.format_exc(), e
+
+            pickle_dump(result, stdout, self._pickle_protocol)
+
+
+class AccessHandle(object):
+    def __init__(self, subprocess, access, id_):
+        self.access = access
+        self._subprocess = subprocess
+        self.id = id_
+
+    def add_subprocess(self, subprocess):
+        self._subprocess = subprocess
+
+    def __repr__(self):
+        try:
+            detail = self.access
+        except AttributeError:
+            detail = '#' + str(self.id)
+        return '<%s of %s>' % (self.__class__.__name__, detail)
+
+    def __getstate__(self):
+        return self.id
+
+    def __setstate__(self, state):
+        self.id = state
+
+    def __getattr__(self, name):
+        if name in ('id', 'access') or name.startswith('_'):
+            raise AttributeError("Something went wrong with unpickling")
+
+        #if not is_py3: print >> sys.stderr, name
+        #print('getattr', name, file=sys.stderr)
+        return partial(self._workaround, force_unicode(name))
+
+    def _workaround(self, name, *args, **kwargs):
+        """
+        TODO Currently we're passing slice objects around. This should not
+        happen. They are also the only unhashable objects that we're passing
+        around.
+        """
+        if args and isinstance(args[0], slice):
+            return self._subprocess.get_compiled_method_return(self.id, name, *args, **kwargs)
+        return self._cached_results(name, *args, **kwargs)
+
+    @memoize_method
+    def _cached_results(self, name, *args, **kwargs):
+        #if type(self._subprocess) == EvaluatorSubprocess:
+            #print(name, args, kwargs,
+                #self._subprocess.get_compiled_method_return(self.id, name, *args, **kwargs)
+            #)
+        return self._subprocess.get_compiled_method_return(self.id, name, *args, **kwargs)

jedi/evaluate/compiled/subprocess/__main__.py

@@ -0,0 +1,55 @@
+import os
+import sys
+
+
+def _get_paths():
+    # Get the path to jedi.
+    _d = os.path.dirname
+    _jedi_path = _d(_d(_d(_d(_d(__file__)))))
+    _parso_path = sys.argv[1]
+    # The paths are the directory that jedi and parso lie in.
+    return {'jedi': _jedi_path, 'parso': _parso_path}
+
+
+# Remove the first entry, because it's simply a directory entry that equals
+# this directory.
+del sys.path[0]
+
+if sys.version_info > (3, 4):
+    from importlib.machinery import PathFinder
+
+    class _ExactImporter(object):
+        def __init__(self, path_dct):
+            self._path_dct = path_dct
+
+        def find_module(self, fullname, path=None):
+            if path is None and fullname in self._path_dct:
+                p = self._path_dct[fullname]
+                loader = PathFinder.find_module(fullname, path=[p])
+                return loader
+            return None
+
+    # Try to import jedi/parso.
+    sys.meta_path.insert(0, _ExactImporter(_get_paths()))
+    from jedi.evaluate.compiled import subprocess  # NOQA
+    sys.meta_path.pop(0)
+else:
+    import imp
+
+    def load(name):
+        paths = list(_get_paths().values())
+        fp, pathname, description = imp.find_module(name, paths)
+        return imp.load_module(name, fp, pathname, description)
+
+    load('parso')
+    load('jedi')
+    from jedi.evaluate.compiled import subprocess  # NOQA
+
+from jedi._compatibility import highest_pickle_protocol  # noqa: E402
+
+
+# Retrieve the pickle protocol.
+host_sys_version = [int(x) for x in sys.argv[2].split('.')]
+pickle_protocol = highest_pickle_protocol([sys.version_info, host_sys_version])
+# And finally start the client.
+subprocess.Listener(pickle_protocol=pickle_protocol).listen()

jedi/evaluate/compiled/subprocess/functions.py

@@ -0,0 +1,113 @@
+import sys
+import os
+
+from jedi._compatibility import find_module, cast_path, force_unicode, \
+    iter_modules, all_suffixes, print_to_stderr
+from jedi.evaluate.compiled import access
+from jedi import parser_utils
+
+
+def get_sys_path():
+    return list(map(cast_path, sys.path))
+
+
+def load_module(evaluator, **kwargs):
+    return access.load_module(evaluator, **kwargs)
+
+
+def get_compiled_method_return(evaluator, id, attribute, *args, **kwargs):
+    handle = evaluator.compiled_subprocess.get_access_handle(id)
+    return getattr(handle.access, attribute)(*args, **kwargs)
+
+
+def get_special_object(evaluator, identifier):
+    return access.get_special_object(evaluator, identifier)
+
+
+def create_simple_object(evaluator, obj):
+    return access.create_access_path(evaluator, obj)
+
+
+def get_module_info(evaluator, sys_path=None, full_name=None, **kwargs):
+    if sys_path is not None:
+        sys.path, temp = sys_path, sys.path
+    try:
+        module_file, module_path, is_pkg = find_module(full_name=full_name, **kwargs)
+    except ImportError:
+        return None, None, None
+    finally:
+        if sys_path is not None:
+            sys.path = temp
+
+    code = None
+    if is_pkg:
+        # In this case, we don't have a file yet. Search for the
+        # __init__ file.
+        if module_path.endswith(('.zip', '.egg')):
+            code = module_file.loader.get_source(full_name)
+        else:
+            module_path = _get_init_path(module_path)
+    elif module_file:
+        if module_path.endswith(('.zip', '.egg')):
+            # Unfortunately we are reading unicode here already, not byes.
+            # It seems however hard to get bytes, because the zip importer
+            # logic just unpacks the zip file and returns a file descriptor
+            # that we cannot as easily access. Therefore we just read it as
+            # a string.
+            code = module_file.read()
+        else:
+            # Read the code with a binary file, because the binary file
+            # might not be proper unicode. This is handled by the parser
+            # wrapper.
+            with open(module_path, 'rb') as f:
+                code = f.read()
+
+        module_file.close()
+
+    return code, cast_path(module_path), is_pkg
+
+
+def list_module_names(evaluator, search_path):
+    return [
+        force_unicode(name)
+        for module_loader, name, is_pkg in iter_modules(search_path)
+    ]
+
+
+def get_builtin_module_names(evaluator):
+    return list(map(force_unicode, sys.builtin_module_names))
+
+
+def _test_raise_error(evaluator, exception_type):
+    """
+    Raise an error to simulate certain problems for unit tests.
+    """
+    raise exception_type
+
+
+def _test_print(evaluator, stderr=None, stdout=None):
+    """
+    Force some prints in the subprocesses. This exists for unit tests.
+    """
+    if stderr is not None:
+        print_to_stderr(stderr)
+        sys.stderr.flush()
+    if stdout is not None:
+        print(stdout)
+        sys.stdout.flush()
+
+
+def _get_init_path(directory_path):
+    """
+    The __init__ file can be searched in a directory. If found return it, else
+    None.
+    """
+    for suffix in all_suffixes():
+        path = os.path.join(directory_path, '__init__' + suffix)
+        if os.path.exists(path):
+            return path
+    return None
+
+
+def safe_literal_eval(evaluator, value):
+    return parser_utils.safe_literal_eval(value)

jedi/evaluate/context.py

@@ -1,206 +0,0 @@
-from jedi._compatibility import Python3Method
-from jedi.common import unite
-from parso.python.tree import ExprStmt, CompFor
-from jedi.parser_utils import clean_scope_docstring, get_doc_with_call_signature
-
-
-class Context(object):
-    """
-    Should be defined, otherwise the API returns empty types.
-    """
-
-    """
-    To be defined by subclasses.
-    """
-    predefined_names = {}
-    tree_node = None
-
-    def __init__(self, evaluator, parent_context=None):
-        self.evaluator = evaluator
-        self.parent_context = parent_context
-
-    @property
-    def api_type(self):
-        # By default just lower name of the class. Can and should be
-        # overwritten.
-        return self.__class__.__name__.lower()
-
-    def get_root_context(self):
-        context = self
-        while True:
-            if context.parent_context is None:
-                return context
-            context = context.parent_context
-
-    def execute(self, arguments):
-        return self.evaluator.execute(self, arguments)
-
-    def execute_evaluated(self, *value_list):
-        """
-        Execute a function with already executed arguments.
-        """
-        from jedi.evaluate.param import ValuesArguments
-        arguments = ValuesArguments([[value] for value in value_list])
-        return self.execute(arguments)
-
-    def eval_node(self, node):
-        return self.evaluator.eval_element(self, node)
-
-    def eval_stmt(self, stmt, seek_name=None):
-        return self.evaluator.eval_statement(self, stmt, seek_name)
-
-    def eval_trailer(self, types, trailer):
-        return self.evaluator.eval_trailer(self, types, trailer)
-
-    @Python3Method
-    def py__getattribute__(self, name_or_str, name_context=None, position=None,
-                           search_global=False, is_goto=False,
-                           analysis_errors=True):
-        if name_context is None:
-            name_context = self
-        return self.evaluator.find_types(
-            self, name_or_str, name_context, position, search_global, is_goto,
-            analysis_errors)
-
-    def create_context(self, node, node_is_context=False, node_is_object=False):
-        return self.evaluator.create_context(self, node, node_is_context, node_is_object)
-
-    def is_class(self):
-        return False
-
-    def py__bool__(self):
-        """
-        Since Wrapper is a super class for classes, functions and modules,
-        the return value will always be true.
-        """
-        return True
-
-    def py__doc__(self, include_call_signature=False):
-        try:
-            self.tree_node.get_doc_node
-        except AttributeError:
-            return ''
-        else:
-            if include_call_signature:
-                return get_doc_with_call_signature(self.tree_node)
-            else:
-                return clean_scope_docstring(self.tree_node)
-        return None
-
-
-class TreeContext(Context):
-    def __init__(self, evaluator, parent_context=None):
-        super(TreeContext, self).__init__(evaluator, parent_context)
-        self.predefined_names = {}
-
-    def __repr__(self):
-        return '<%s: %s>' % (self.__class__.__name__, self.tree_node)
-
-
-class AbstractLazyContext(object):
-    def __init__(self, data):
-        self.data = data
-
-    def __repr__(self):
-        return '<%s: %s>' % (self.__class__.__name__, self.data)
-
-    def infer(self):
-        raise NotImplementedError
-
-
-class LazyKnownContext(AbstractLazyContext):
-    """data is a context."""
-    def infer(self):
-        return set([self.data])
-
-
-class LazyKnownContexts(AbstractLazyContext):
-    """data is a set of contexts."""
-    def infer(self):
-        return self.data
-
-
-class LazyUnknownContext(AbstractLazyContext):
-    def __init__(self):
-        super(LazyUnknownContext, self).__init__(None)
-
-    def infer(self):
-        return set()
-
-
-class LazyTreeContext(AbstractLazyContext):
-    def __init__(self, context, node):
-        super(LazyTreeContext, self).__init__(node)
-        self._context = context
-        # We need to save the predefined names. It's an unfortunate side effect
-        # that needs to be tracked otherwise results will be wrong.
-        self._predefined_names = dict(context.predefined_names)
-
-    def infer(self):
-        old, self._context.predefined_names = \
-            self._context.predefined_names, self._predefined_names
-        try:
-            return self._context.eval_node(self.data)
-        finally:
-            self._context.predefined_names = old
-
-
-def get_merged_lazy_context(lazy_contexts):
-    if len(lazy_contexts) > 1:
-        return MergedLazyContexts(lazy_contexts)
-    else:
-        return lazy_contexts[0]
-
-
-class MergedLazyContexts(AbstractLazyContext):
-    """data is a list of lazy contexts."""
-    def infer(self):
-        return unite(l.infer() for l in self.data)
-
-
-class ContextualizedNode(object):
-    def __init__(self, context, node):
-        self.context = context
-        self._node = node
-
-    def get_root_context(self):
-        return self.context.get_root_context()
-
-    def infer(self):
-        return self.context.eval_node(self._node)
-
-
-class ContextualizedName(ContextualizedNode):
-    # TODO merge with TreeNameDefinition?!
-    @property
-    def name(self):
-        return self._node
-
-    def assignment_indexes(self):
-        """
-        Returns an array of tuple(int, node) of the indexes that are used in
-        tuple assignments.
-
-        For example if the name is ``y`` in the following code::
-
-            x, (y, z) = 2, ''
-
-        would result in ``[(1, xyz_node), (0, yz_node)]``.
-        """
-        indexes = []
-        node = self._node.parent
-        compare = self._node
-        while node is not None:
-            if node.type in ('testlist', 'testlist_comp', 'testlist_star_expr', 'exprlist'):
-                for i, child in enumerate(node.children):
-                    if child == compare:
-                        indexes.insert(0, (int(i / 2), node))
-                        break
-                else:
-                    raise LookupError("Couldn't find the assignment.")
-            elif isinstance(node, (ExprStmt, CompFor)):
-                break
-
-            compare = node
-            node = node.parent
-        return indexes

jedi/evaluate/context/__init__.py

@@ -0,0 +1,5 @@
+from jedi.evaluate.context.module import ModuleContext
+from jedi.evaluate.context.klass import ClassContext
+from jedi.evaluate.context.function import FunctionContext, FunctionExecutionContext
+from jedi.evaluate.context.instance import AnonymousInstance, BoundMethod, \
+    CompiledInstance, AbstractInstanceContext, TreeInstance

jedi/evaluate/context/asynchronous.py

@@ -0,0 +1,38 @@
+from jedi.evaluate.filters import publish_method, BuiltinOverwrite
+from jedi.evaluate.base_context import ContextSet
+
+
+class AsyncBase(BuiltinOverwrite):
+    def __init__(self, evaluator, func_execution_context):
+        super(AsyncBase, self).__init__(evaluator)
+        self.func_execution_context = func_execution_context
+
+    @property
+    def name(self):
+        return self.get_object().name
+
+    def __repr__(self):
+        return "<%s of %s>" % (type(self).__name__, self.func_execution_context)
+
+
+class Coroutine(AsyncBase):
+    special_object_identifier = u'COROUTINE'
+
+    @publish_method('__await__')
+    def _await(self):
+        return ContextSet(CoroutineWrapper(self.evaluator, self.func_execution_context))
+
+
+class CoroutineWrapper(AsyncBase):
+    special_object_identifier = u'COROUTINE_WRAPPER'
+
+    def py__stop_iteration_returns(self):
+        return self.func_execution_context.get_return_values()
+
+
+class AsyncGenerator(AsyncBase):
+    """Handling of `yield` functions."""
+    special_object_identifier = u'ASYNC_GENERATOR'
+
+    def py__aiter__(self):
+        return self.func_execution_context.get_yield_lazy_contexts(is_async=True)

jedi/evaluate/context/function.py

@@ -0,0 +1,253 @@
+from parso.python import tree
+
+from jedi._compatibility import use_metaclass
+from jedi import debug
+from jedi.evaluate.cache import evaluator_method_cache, CachedMetaClass
+from jedi.evaluate import compiled
+from jedi.evaluate import recursion
+from jedi.evaluate import docstrings
+from jedi.evaluate import pep0484
+from jedi.evaluate import flow_analysis
+from jedi.evaluate import helpers
+from jedi.evaluate.arguments import AnonymousArguments
+from jedi.evaluate.filters import ParserTreeFilter, FunctionExecutionFilter, \
+    ContextName, AbstractNameDefinition, ParamName
+from jedi.evaluate.base_context import ContextualizedNode, NO_CONTEXTS, \
+    ContextSet, TreeContext
+from jedi.evaluate.lazy_context import LazyKnownContexts, LazyKnownContext, \
+    LazyTreeContext
+from jedi.evaluate.context import iterable
+from jedi.evaluate.context import asynchronous
+from jedi import parser_utils
+from jedi.evaluate.parser_cache import get_yield_exprs
+
+
+class LambdaName(AbstractNameDefinition):
+    string_name = '<lambda>'
+    api_type = u'function'
+
+    def __init__(self, lambda_context):
+        self._lambda_context = lambda_context
+        self.parent_context = lambda_context.parent_context
+
+    @property
+    def start_pos(self):
+        return self._lambda_context.tree_node.start_pos
+
+    def infer(self):
+        return ContextSet(self._lambda_context)
+
+
+class AbstractFunction(TreeContext):
+    api_type = u'function'
+
+    def get_filters(self, search_global, until_position=None, origin_scope=None):
+        if search_global:
+            yield ParserTreeFilter(
+                self.evaluator,
+                context=self,
+                until_position=until_position,
+                origin_scope=origin_scope
+            )
+        else:
+            scope = self.py__class__()
+            for filter in scope.get_filters(search_global=False, origin_scope=origin_scope):
+                yield filter
+
+    def get_param_names(self):
+        function_execution = self.get_function_execution()
+        return [ParamName(function_execution, param.name)
+                for param in self.tree_node.get_params()]
+
+    @property
+    def name(self):
+        if self.tree_node.type == 'lambdef':
+            return LambdaName(self)
+        return ContextName(self, self.tree_node.name)
+
+    def get_function_execution(self, arguments=None):
+        raise NotImplementedError
+
+    def py__call__(self, arguments):
+        function_execution = self.get_function_execution(arguments)
+        return self.infer_function_execution(function_execution)
+
+    def infer_function_execution(self, function_execution):
+        """
+        Created to be used by inheritance.
+        """
+        is_coroutine = self.tree_node.parent.type == 'async_stmt'
+        is_generator = bool(get_yield_exprs(self.evaluator, self.tree_node))
+
+        if is_coroutine:
+            if is_generator:
+                if self.evaluator.environment.version_info < (3, 6):
+                    return NO_CONTEXTS
+                return ContextSet(asynchronous.AsyncGenerator(self.evaluator, function_execution))
+            else:
+                if self.evaluator.environment.version_info < (3, 5):
+                    return NO_CONTEXTS
+                return ContextSet(asynchronous.Coroutine(self.evaluator, function_execution))
+        else:
+            if is_generator:
+                return ContextSet(iterable.Generator(self.evaluator, function_execution))
+            else:
+                return function_execution.get_return_values()
+
+    def py__name__(self):
+        return self.name.string_name
+
+
+class FunctionContext(use_metaclass(CachedMetaClass, AbstractFunction)):
+    """
+    Needed because of decorators. Decorators are evaluated here.
+    """
+    @classmethod
+    def from_context(cls, context, tree_node):
+        from jedi.evaluate.context import AbstractInstanceContext
+
+        while context.is_class() or isinstance(context, AbstractInstanceContext):
+            context = context.parent_context
+
+        return cls(context.evaluator, parent_context=context, tree_node=tree_node)
+
+    def get_function_execution(self, arguments=None):
+        if arguments is None:
+            arguments = AnonymousArguments()
+
+        return FunctionExecutionContext(self.evaluator, self.parent_context, self, arguments)
+
+    def py__class__(self):
+        return compiled.get_special_object(self.evaluator, u'FUNCTION_CLASS')
+
+
+class FunctionExecutionContext(TreeContext):
+    """
+    This class is used to evaluate functions and their returns.
+
+    This is the most complicated class, because it contains the logic to
+    transfer parameters. It is even more complicated, because there may be
+    multiple calls to functions and recursion has to be avoided. But this is
+    responsibility of the decorators.
+    """
+    function_execution_filter = FunctionExecutionFilter
+
+    def __init__(self, evaluator, parent_context, function_context, var_args):
+        super(FunctionExecutionContext, self).__init__(
+            evaluator,
+            parent_context,
+            function_context.tree_node,
+        )
+        self.function_context = function_context
+        self.var_args = var_args
+
+    @evaluator_method_cache(default=NO_CONTEXTS)
+    @recursion.execution_recursion_decorator()
+    def get_return_values(self, check_yields=False):
+        funcdef = self.tree_node
+        if funcdef.type == 'lambdef':
+            return self.eval_node(funcdef.children[-1])
+
+        if check_yields:
+            context_set = NO_CONTEXTS
+            returns = get_yield_exprs(self.evaluator, funcdef)
+        else:
+            returns = funcdef.iter_return_stmts()
+            context_set = docstrings.infer_return_types(self.function_context)
+            context_set |= pep0484.infer_return_types(self.function_context)
+
+        for r in returns:
+            check = flow_analysis.reachability_check(self, funcdef, r)
+            if check is flow_analysis.UNREACHABLE:
+                debug.dbg('Return unreachable: %s', r)
+            else:
+                if check_yields:
+                    context_set |= ContextSet.from_sets(
+                        lazy_context.infer()
+                        for lazy_context in self._get_yield_lazy_context(r)
+                    )
+                else:
+                    try:
+                        children = r.children
+                    except AttributeError:
+                        ctx = compiled.builtin_from_name(self.evaluator, u'None')
+                        context_set |= ContextSet(ctx)
+                    else:
+                        context_set |= self.eval_node(children[1])
+            if check is flow_analysis.REACHABLE:
+                debug.dbg('Return reachable: %s', r)
+                break
+        return context_set
+
+    def _get_yield_lazy_context(self, yield_expr):
+        if yield_expr.type == 'keyword':
+            # `yield` just yields None.
+            ctx = compiled.builtin_from_name(self.evaluator, u'None')
+            yield LazyKnownContext(ctx)
+            return
+
+        node = yield_expr.children[1]
+        if node.type == 'yield_arg':  # It must be a yield from.
+            cn = ContextualizedNode(self, node.children[1])
+            for lazy_context in cn.infer().iterate(cn):
+                yield lazy_context
+        else:
+            yield LazyTreeContext(self, node)
+
+    @recursion.execution_recursion_decorator(default=iter([]))
+    def get_yield_lazy_contexts(self, is_async=False):
+        # TODO: if is_async, wrap yield statements in Awaitable/async_generator_asend
+        for_parents = [(y, tree.search_ancestor(y, 'for_stmt', 'funcdef',
+                                                'while_stmt', 'if_stmt'))
+                       for y in get_yield_exprs(self.evaluator, self.tree_node)]
+
+        # Calculate if the yields are placed within the same for loop.
+        yields_order = []
+        last_for_stmt = None
+        for yield_, for_stmt in for_parents:
+            # For really simple for loops we can predict the order. Otherwise
+            # we just ignore it.
+            parent = for_stmt.parent
+            if parent.type == 'suite':
+                parent = parent.parent
+            if for_stmt.type == 'for_stmt' and parent == self.tree_node \
+                    and parser_utils.for_stmt_defines_one_name(for_stmt):  # Simplicity for now.
+                if for_stmt == last_for_stmt:
+                    yields_order[-1][1].append(yield_)
+                else:
+                    yields_order.append((for_stmt, [yield_]))
+            elif for_stmt == self.tree_node:
+                yields_order.append((None, [yield_]))
+            else:
+                types = self.get_return_values(check_yields=True)
+                if types:
+                    yield LazyKnownContexts(types)
+                return
+            last_for_stmt = for_stmt
+
+        for for_stmt, yields in yields_order:
+            if for_stmt is None:
+                # No for_stmt, just normal yields.
+                for yield_ in yields:
+                    for result in self._get_yield_lazy_context(yield_):
+                        yield result
+            else:
+                input_node = for_stmt.get_testlist()
+                cn = ContextualizedNode(self, input_node)
+                ordered = cn.infer().iterate(cn)
+                ordered = list(ordered)
+                for lazy_context in ordered:
+                    dct = {str(for_stmt.children[1].value): lazy_context.infer()}
+                    with helpers.predefine_names(self, for_stmt, dct):
+                        for yield_in_same_for_stmt in yields:
+                            for result in self._get_yield_lazy_context(yield_in_same_for_stmt):
+                                yield result
+
+    def get_filters(self, search_global, until_position=None, origin_scope=None):
+        yield self.function_execution_filter(self.evaluator, self,
+                                             until_position=until_position,
+                                             origin_scope=origin_scope)
+
+    @evaluator_method_cache()
+    def get_executed_params(self):
+        return self.var_args.get_executed_params(self)

jedi/evaluate/context/instance.py

@@ -1,44 +1,55 @@
 from abc import abstractproperty
 
-from jedi._compatibility import is_py3
-from jedi.common import unite
 from jedi import debug
+from jedi import settings
 from jedi.evaluate import compiled
 from jedi.evaluate import filters
-from jedi.evaluate.context import Context, LazyKnownContext, LazyKnownContexts
+from jedi.evaluate.base_context import Context, NO_CONTEXTS, ContextSet, \
+    iterator_to_context_set
+from jedi.evaluate.lazy_context import LazyKnownContext, LazyKnownContexts
 from jedi.evaluate.cache import evaluator_method_cache
-from jedi.evaluate.param import AbstractArguments, AnonymousArguments
-from jedi.cache import memoize_method
-from jedi.evaluate import representation as er
-from jedi.evaluate import iterable
+from jedi.evaluate.arguments import AbstractArguments, AnonymousArguments
+from jedi.evaluate.context.function import FunctionExecutionContext, \
+    FunctionContext, AbstractFunction
+from jedi.evaluate.context.klass import ClassContext, apply_py__get__, ClassFilter
+from jedi.evaluate.context import iterable
 from jedi.parser_utils import get_parent_scope
 
 
+class InstanceExecutedParam(object):
+    def __init__(self, instance):
+        self._instance = instance
 
-class InstanceFunctionExecution(er.FunctionExecutionContext):
-    def __init__(self, instance, parent_context, function_context, var_args):
-        self.instance = instance
-        var_args = InstanceVarArgs(self, var_args)
-
-        super(InstanceFunctionExecution, self).__init__(
-            instance.evaluator, parent_context, function_context, var_args)
+    def infer(self):
+        return ContextSet(self._instance)
 
 
-class AnonymousInstanceFunctionExecution(er.FunctionExecutionContext):
-    function_execution_filter = filters.AnonymousInstanceFunctionExecutionFilter
+class AnonymousInstanceArguments(AnonymousArguments):
+    def __init__(self, instance):
+        self._instance = instance
 
-    def __init__(self, instance, parent_context, function_context, var_args):
-        self.instance = instance
-        super(AnonymousInstanceFunctionExecution, self).__init__(
-            instance.evaluator, parent_context, function_context, var_args)
+    def get_executed_params(self, execution_context):
+        from jedi.evaluate.dynamic import search_params
+        self_param = InstanceExecutedParam(self._instance)
+        tree_params = execution_context.tree_node.get_params()
+        if len(tree_params) == 1:
+            # If the only param is self, we don't need to try to find
+            # executions of this function, we have all the params already.
+            return [self_param]
+        executed_params = list(search_params(
+            execution_context.evaluator,
+            execution_context,
+            execution_context.tree_node
+        ))
+        executed_params[0] = self_param
+        return executed_params
 
 
 class AbstractInstanceContext(Context):
     """
     This class is used to evaluate instances.
     """
-    api_type = 'instance'
-    function_execution_cls = InstanceFunctionExecution
+    api_type = u'instance'
 
     def __init__(self, evaluator, parent_context, class_context, var_args):
         super(AbstractInstanceContext, self).__init__(evaluator, parent_context)
@@ -52,13 +63,13 @@ class AbstractInstanceContext(Context):
 
     @property
     def py__call__(self):
-        names = self.get_function_slot_names('__call__')
+        names = self.get_function_slot_names(u'__call__')
         if not names:
             # Means the Instance is not callable.
             raise AttributeError
 
         def execute(arguments):
-            return unite(name.execute(arguments) for name in names)
+            return ContextSet.from_sets(name.infer().execute(arguments) for name in names)
 
         return execute
 
@@ -80,36 +91,34 @@ class AbstractInstanceContext(Context):
         return []
 
     def execute_function_slots(self, names, *evaluated_args):
-        return unite(
-            name.execute_evaluated(*evaluated_args)
+        return ContextSet.from_sets(
+            name.infer().execute_evaluated(*evaluated_args)
             for name in names
         )
 
     def py__get__(self, obj):
         # Arguments in __get__ descriptors are obj, class.
         # `method` is the new parent of the array, don't know if that's good.
-        names = self.get_function_slot_names('__get__')
+        names = self.get_function_slot_names(u'__get__')
         if names:
             if isinstance(obj, AbstractInstanceContext):
                 return self.execute_function_slots(names, obj, obj.class_context)
             else:
-                none_obj = compiled.create(self.evaluator, None)
+                none_obj = compiled.builtin_from_name(self.evaluator, u'None')
                 return self.execute_function_slots(names, none_obj, obj)
         else:
-            return set([self])
+            return ContextSet(self)
 
     def get_filters(self, search_global=None, until_position=None,
                     origin_scope=None, include_self_names=True):
         if include_self_names:
             for cls in self.class_context.py__mro__():
-                if isinstance(cls, compiled.CompiledObject):
-                    if cls.tree_node is not None:
-                        # In this case we're talking about a fake object, it
-                        # doesn't make sense for normal compiled objects to
-                        # search for self variables.
-                        yield SelfNameFilter(self.evaluator, self, cls, origin_scope)
-                else:
-                    yield SelfNameFilter(self.evaluator, self, cls, origin_scope)
+                if not isinstance(cls, compiled.CompiledObject) \
+                        or cls.tree_node is not None:
+                    # In this case we're excluding compiled objects that are
+                    # not fake objects. It doesn't make sense for normal
+                    # compiled objects to search for self variables.
+                    yield SelfAttributeFilter(self.evaluator, self, cls, origin_scope)
 
         for cls in self.class_context.py__mro__():
             if isinstance(cls, compiled.CompiledObject):
@@ -119,16 +128,16 @@ class AbstractInstanceContext(Context):
 
     def py__getitem__(self, index):
         try:
-            names = self.get_function_slot_names('__getitem__')
+            names = self.get_function_slot_names(u'__getitem__')
         except KeyError:
             debug.warning('No __getitem__, cannot access the array.')
-            return set()
+            return NO_CONTEXTS
         else:
-            index_obj = compiled.create(self.evaluator, index)
+            index_obj = compiled.create_simple_object(self.evaluator, index)
             return self.execute_function_slots(names, index_obj)
 
     def py__iter__(self):
-        iter_slot_names = self.get_function_slot_names('__iter__')
+        iter_slot_names = self.get_function_slot_names(u'__iter__')
         if not iter_slot_names:
             debug.warning('No __iter__ on %s.' % self)
             return
@@ -136,7 +145,10 @@ class AbstractInstanceContext(Context):
         for generator in self.execute_function_slots(iter_slot_names):
             if isinstance(generator, AbstractInstanceContext):
                 # `__next__` logic.
-                name = '__next__' if is_py3 else 'next'
+                if self.evaluator.environment.version_info.major == 2:
+                    name = u'next'
+                else:
+                    name = u'__next__'
                 iter_slot_names = generator.get_function_slot_names(name)
                 if iter_slot_names:
                     yield LazyKnownContexts(
@@ -152,21 +164,18 @@ class AbstractInstanceContext(Context):
     def name(self):
         pass
 
-    def _create_init_execution(self, class_context, func_node):
-        bound_method = BoundMethod(
-            self.evaluator, self, class_context, self.parent_context, func_node
-        )
-        return self.function_execution_cls(
-            self,
-            class_context.parent_context,
-            bound_method,
-            self.var_args
-        )
+    def _create_init_execution(self, class_context, bound_method):
+        return bound_method.get_function_execution(self.var_args)
 
     def create_init_executions(self):
-        for name in self.get_function_slot_names('__init__'):
-            if isinstance(name, LazyInstanceName):
-                yield self._create_init_execution(name.class_context, name.tree_name.parent)
+        for name in self.get_function_slot_names(u'__init__'):
+            if isinstance(name, LazyInstanceClassName):
+                function = FunctionContext.from_context(
+                    self.parent_context,
+                    name.tree_name.parent
+                )
+                bound_method = BoundMethod(self, name.class_context, function)
+                yield self._create_init_execution(name.class_context, bound_method)
 
     @evaluator_method_cache()
     def create_instance_context(self, class_context, node):
@@ -178,16 +187,17 @@ class AbstractInstanceContext(Context):
         else:
             parent_context = self.create_instance_context(class_context, scope)
             if scope.type == 'funcdef':
+                func = FunctionContext.from_context(
+                    parent_context,
+                    scope,
+                )
+                bound_method = BoundMethod(self, class_context, func)
                 if scope.name.value == '__init__' and parent_context == class_context:
-                    return self._create_init_execution(class_context, scope)
+                    return self._create_init_execution(class_context, bound_method)
                 else:
-                    bound_method = BoundMethod(
-                        self.evaluator, self, class_context,
-                        parent_context, scope
-                    )
                     return bound_method.get_function_execution()
             elif scope.type == 'classdef':
-                class_context = er.ClassContext(self.evaluator, scope, parent_context)
+                class_context = ClassContext(self.evaluator, parent_context, scope)
                 return class_context
             elif scope.type == 'comp_for':
                 # Comprehensions currently don't have a special scope in Jedi.
@@ -202,14 +212,18 @@ class AbstractInstanceContext(Context):
 
 
 class CompiledInstance(AbstractInstanceContext):
-    def __init__(self, *args, **kwargs):
-        super(CompiledInstance, self).__init__(*args, **kwargs)
+    def __init__(self, evaluator, parent_context, class_context, var_args):
+        self._original_var_args = var_args
+
         # I don't think that dynamic append lookups should happen here. That
         # sounds more like something that should go to py__iter__.
-        if self.class_context.name.string_name in ['list', 'set'] \
-                and self.parent_context.get_root_context() == self.evaluator.BUILTINS:
+        if class_context.py__name__() in ['list', 'set'] \
+                and parent_context.get_root_context() == evaluator.builtins_module:
             # compare the module path with the builtin name.
-            self.var_args = iterable.get_dynamic_array_instance(self)
+            if settings.dynamic_array_additions:
+                var_args = iterable.get_dynamic_array_instance(self, var_args)
+
+        super(CompiledInstance, self).__init__(evaluator, parent_context, class_context, var_args)
 
     @property
     def name(self):
@@ -221,6 +235,13 @@ class CompiledInstance(AbstractInstanceContext):
         else:
             return super(CompiledInstance, self).create_instance_context(class_context, node)
 
+    def get_first_non_keyword_argument_contexts(self):
+        key, lazy_context = next(self._original_var_args.unpack(), ('', None))
+        if key is not None:
+            return NO_CONTEXTS
+
+        return lazy_context.infer()
+
 
 class TreeInstance(AbstractInstanceContext):
     def __init__(self, evaluator, parent_context, class_context, var_args):
@@ -234,89 +255,105 @@ class TreeInstance(AbstractInstanceContext):
 
 
 class AnonymousInstance(TreeInstance):
-    function_execution_cls = AnonymousInstanceFunctionExecution
-
     def __init__(self, evaluator, parent_context, class_context):
         super(AnonymousInstance, self).__init__(
             evaluator,
             parent_context,
             class_context,
-            var_args=AnonymousArguments(),
+            var_args=AnonymousInstanceArguments(self),
         )
 
 
 class CompiledInstanceName(compiled.CompiledName):
-    def __init__(self, evaluator, instance, parent_context, name):
-        super(CompiledInstanceName, self).__init__(evaluator, parent_context, name)
+
+    def __init__(self, evaluator, instance, klass, name):
+        super(CompiledInstanceName, self).__init__(
+            evaluator,
+            klass.parent_context,
+            name.string_name
+        )
         self._instance = instance
+        self._class = klass
+        self._class_member_name = name
 
+    @iterator_to_context_set
     def infer(self):
-        for result_context in super(CompiledInstanceName, self).infer():
-            if isinstance(result_context, er.FunctionContext):
-                parent_context = result_context.parent_context
-                while parent_context.is_class():
-                    parent_context = parent_context.parent_context
-
-                yield BoundMethod(
-                    result_context.evaluator, self._instance, self.parent_context,
-                    parent_context, result_context.tree_node
-                )
+        for result_context in self._class_member_name.infer():
+            is_function = result_context.api_type == 'function'
+            if result_context.tree_node is not None and is_function:
+                yield BoundMethod(self._instance, self._class, result_context)
             else:
-                if result_context.api_type == 'function':
+                if is_function:
                     yield CompiledBoundMethod(result_context)
                 else:
                     yield result_context
 
 
-class CompiledInstanceClassFilter(compiled.CompiledObjectFilter):
+class CompiledInstanceClassFilter(filters.AbstractFilter):
     name_class = CompiledInstanceName
 
-    def __init__(self, evaluator, instance, compiled_object):
-        super(CompiledInstanceClassFilter, self).__init__(
-            evaluator,
-            compiled_object,
-            is_instance=True,
+    def __init__(self, evaluator, instance, klass):
+        self._evaluator = evaluator
+        self._instance = instance
+        self._class = klass
+        self._class_filter = next(klass.get_filters(is_instance=True))
+
+    def get(self, name):
+        return self._convert(self._class_filter.get(name))
+
+    def values(self):
+        return self._convert(self._class_filter.values())
+
+    def _convert(self, names):
+        return [
+            CompiledInstanceName(self._evaluator, self._instance, self._class, n)
+            for n in names
+        ]
+
+
+class BoundMethod(AbstractFunction):
+    def __init__(self, instance, klass, function):
+        super(BoundMethod, self).__init__(
+            function.evaluator,
+            function.parent_context,
+            function.tree_node,
         )
         self._instance = instance
+        self._class = klass
+        self._function = function
 
-    def _create_name(self, name):
-        return self.name_class(
-            self._evaluator, self._instance, self._compiled_object, name)
-
-
-class BoundMethod(er.FunctionContext):
-    def __init__(self, evaluator, instance, class_context, *args, **kwargs):
-        super(BoundMethod, self).__init__(evaluator, *args, **kwargs)
-        self._instance = instance
-        self._class_context = class_context
+    def py__class__(self):
+        return compiled.get_special_object(self.evaluator, u'BOUND_METHOD_CLASS')
 
     def get_function_execution(self, arguments=None):
         if arguments is None:
-            arguments = AnonymousArguments()
-            return AnonymousInstanceFunctionExecution(
-                self._instance, self.parent_context, self, arguments)
-        else:
-            return InstanceFunctionExecution(
-                self._instance, self.parent_context, self, arguments)
+            arguments = AnonymousInstanceArguments(self._instance)
+
+        arguments = InstanceArguments(self._instance, arguments)
+
+        if isinstance(self._function, compiled.CompiledObject):
+            # This is kind of weird, because it's coming from a compiled object
+            # and we're not sure if we want that in the future.
+            return FunctionExecutionContext(
+                self.evaluator, self.parent_context, self, arguments
+            )
+
+        return self._function.get_function_execution(arguments)
+
+    def __repr__(self):
+        return '<%s: %s>' % (self.__class__.__name__, self._function)
 
 
 class CompiledBoundMethod(compiled.CompiledObject):
     def __init__(self, func):
         super(CompiledBoundMethod, self).__init__(
-            func.evaluator, func.obj, func.parent_context, func.tree_node)
+            func.evaluator, func.access_handle, func.parent_context, func.tree_node)
 
     def get_param_names(self):
         return list(super(CompiledBoundMethod, self).get_param_names())[1:]
 
 
-class InstanceNameDefinition(filters.TreeNameDefinition):
-    def infer(self):
-        contexts = super(InstanceNameDefinition, self).infer()
-        for context in contexts:
-            yield context
-
-
-class LazyInstanceName(filters.TreeNameDefinition):
+class SelfName(filters.TreeNameDefinition):
     """
     This name calculates the parent_context lazily.
     """
@@ -330,61 +367,69 @@ class LazyInstanceName(filters.TreeNameDefinition):
         return self._instance.create_instance_context(self.class_context, self.tree_name)
 
 
-class LazyInstanceClassName(LazyInstanceName):
+class LazyInstanceClassName(object):
+    def __init__(self, instance, class_context, class_member_name):
+        self._instance = instance
+        self.class_context = class_context
+        self._class_member_name = class_member_name
+
+    @iterator_to_context_set
     def infer(self):
-        for result_context in super(LazyInstanceClassName, self).infer():
-            if isinstance(result_context, er.FunctionContext):
+        for result_context in self._class_member_name.infer():
+            if isinstance(result_context, FunctionContext):
                 # Classes are never used to resolve anything within the
                 # functions. Only other functions and modules will resolve
                 # those things.
-                parent_context = result_context.parent_context
-                while parent_context.is_class():
-                    parent_context = parent_context.parent_context
-
-                yield BoundMethod(
-                    result_context.evaluator, self._instance, self.class_context,
-                    parent_context, result_context.tree_node
-                )
+                yield BoundMethod(self._instance, self.class_context, result_context)
             else:
-                for c in er.apply_py__get__(result_context, self._instance):
+                for c in apply_py__get__(result_context, self._instance):
                     yield c
 
+    def __getattr__(self, name):
+        return getattr(self._class_member_name, name)
 
-class InstanceClassFilter(filters.ParserTreeFilter):
-    name_class = LazyInstanceClassName
+
+class InstanceClassFilter(filters.AbstractFilter):
+    """
+    This filter is special in that it uses the class filter and wraps the
+    resulting names in LazyINstanceClassName. The idea is that the class name
+    filtering can be very flexible and always be reflected in instances.
+    """
+    def __init__(self, evaluator, context, class_context, origin_scope):
+        self._instance = context
+        self._class_context = class_context
+        self._class_filter = next(class_context.get_filters(
+            search_global=False,
+            origin_scope=origin_scope,
+            is_instance=True,
+        ))
+
+    def get(self, name):
+        return self._convert(self._class_filter.get(name))
+
+    def values(self):
+        return self._convert(self._class_filter.values())
+
+    def _convert(self, names):
+        return [LazyInstanceClassName(self._instance, self._class_context, n) for n in names]
+
+
+class SelfAttributeFilter(ClassFilter):
+    """
+    This class basically filters all the use cases where `self.*` was assigned.
+    """
+    name_class = SelfName
 
     def __init__(self, evaluator, context, class_context, origin_scope):
-        super(InstanceClassFilter, self).__init__(
+        super(SelfAttributeFilter, self).__init__(
             evaluator=evaluator,
             context=context,
             node_context=class_context,
-            origin_scope=origin_scope
+            origin_scope=origin_scope,
+            is_instance=True,
         )
         self._class_context = class_context
 
-    def _equals_origin_scope(self):
-        node = self._origin_scope
-        while node is not None:
-            if node == self._parser_scope or node == self.context:
-                return True
-            node = get_parent_scope(node)
-        return False
-
-    def _access_possible(self, name):
-        return not name.value.startswith('__') or name.value.endswith('__') \
-            or self._equals_origin_scope()
-
-    def _filter(self, names):
-        names = super(InstanceClassFilter, self)._filter(names)
-        return [name for name in names if self._access_possible(name)]
-
-    def _convert_names(self, names):
-        return [self.name_class(self.context, self._class_context, name) for name in names]
-
-
-class SelfNameFilter(InstanceClassFilter):
-    name_class = LazyInstanceName
-
     def _filter(self, names):
         names = self._filter_self_names(names)
         if isinstance(self._parser_scope, compiled.CompiledObject) and False:
@@ -403,19 +448,18 @@ class SelfNameFilter(InstanceClassFilter):
                 if name.is_definition() and self._access_possible(name):
                     yield name
 
+    def _convert_names(self, names):
+        return [self.name_class(self.context, self._class_context, name) for name in names]
+
     def _check_flows(self, names):
         return names
 
 
-class InstanceVarArgs(AbstractArguments):
-    def __init__(self, execution_context, var_args):
-        self._execution_context = execution_context
+class InstanceArguments(AbstractArguments):
+    def __init__(self, instance, var_args):
+        self.instance = instance
         self._var_args = var_args
 
-    @memoize_method
-    def _get_var_args(self):
-        return self._var_args
-
     @property
     def argument_node(self):
         return self._var_args.argument_node
@@ -425,9 +469,15 @@ class InstanceVarArgs(AbstractArguments):
         return self._var_args.trailer
 
     def unpack(self, func=None):
-        yield None, LazyKnownContext(self._execution_context.instance)
-        for values in self._get_var_args().unpack(func):
+        yield None, LazyKnownContext(self.instance)
+        for values in self._var_args.unpack(func):
             yield values
 
     def get_calling_nodes(self):
-        return self._get_var_args().get_calling_nodes()
+        return self._var_args.get_calling_nodes()
+
+    def get_executed_params(self, execution_context):
+        if isinstance(self._var_args, AnonymousInstanceArguments):
+            return self._var_args.get_executed_params(execution_context)
+
+        return super(InstanceArguments, self).get_executed_params(execution_context)

jedi/evaluate/context/iterable.py

@@ -22,169 +22,67 @@ It is important to note that:
 """
 from jedi import debug
 from jedi import settings
-from jedi import common
-from jedi.common import unite, safe_property
-from jedi._compatibility import unicode, zip_longest, is_py3
+from jedi._compatibility import force_unicode, is_py3
+from jedi.cache import memoize_method
 from jedi.evaluate import compiled
-from jedi.evaluate import helpers
 from jedi.evaluate import analysis
-from jedi.evaluate import pep0484
-from jedi.evaluate import context
-from jedi.evaluate import precedence
 from jedi.evaluate import recursion
+from jedi.evaluate.lazy_context import LazyKnownContext, LazyKnownContexts, \
+    LazyTreeContext
+from jedi.evaluate.helpers import get_int_or_none, is_string, \
+    predefine_names, evaluate_call_of_leaf, reraise_as_evaluator, \
+    EvaluatorKeyError
+from jedi.evaluate.utils import safe_property
+from jedi.evaluate.utils import to_list
 from jedi.evaluate.cache import evaluator_method_cache
-from jedi.evaluate.filters import DictFilter, AbstractNameDefinition, \
-    ParserTreeFilter
+from jedi.evaluate.filters import ParserTreeFilter, BuiltinOverwrite, \
+    publish_method
+from jedi.evaluate.base_context import ContextSet, NO_CONTEXTS, Context, \
+    TreeContext, ContextualizedNode
 from jedi.parser_utils import get_comp_fors
 
 
-class AbstractSequence(context.Context):
-    builtin_methods = {}
-    api_type = 'instance'
-
-    def __init__(self, evaluator):
-        super(AbstractSequence, self).__init__(evaluator, evaluator.BUILTINS)
-
-    def get_filters(self, search_global, until_position=None, origin_scope=None):
-        raise NotImplementedError
-
-    @property
-    def name(self):
-        return compiled.CompiledContextName(self, self.array_type)
+class IterableMixin(object):
+    def py__stop_iteration_returns(self):
+        return ContextSet(compiled.builtin_from_name(self.evaluator, u'None'))
 
 
-class BuiltinMethod(object):
-    """``Generator.__next__`` ``dict.values`` methods and so on."""
-    def __init__(self, builtin_context, method, builtin_func):
-        self._builtin_context = builtin_context
-        self._method = method
-        self._builtin_func = builtin_func
-
-    def py__call__(self, params):
-        return self._method(self._builtin_context)
-
-    def __getattr__(self, name):
-        return getattr(self._builtin_func, name)
-
-
-class SpecialMethodFilter(DictFilter):
-    """
-    A filter for methods that are defined in this module on the corresponding
-    classes like Generator (for __next__, etc).
-    """
-    class SpecialMethodName(AbstractNameDefinition):
-        api_type = 'function'
-
-        def __init__(self, parent_context, string_name, callable_, builtin_context):
-            self.parent_context = parent_context
-            self.string_name = string_name
-            self._callable = callable_
-            self._builtin_context = builtin_context
-
-        def infer(self):
-            filter = next(self._builtin_context.get_filters())
-            # We can take the first index, because on builtin methods there's
-            # always only going to be one name. The same is true for the
-            # inferred values.
-            builtin_func = next(iter(filter.get(self.string_name)[0].infer()))
-            return set([BuiltinMethod(self.parent_context, self._callable, builtin_func)])
-
-    def __init__(self, context, dct, builtin_context):
-        super(SpecialMethodFilter, self).__init__(dct)
-        self.context = context
-        self._builtin_context = builtin_context
-        """
-        This context is what will be used to introspect the name, where as the
-        other context will be used to execute the function.
-
-        We distinguish, because we have to.
-        """
-
-    def _convert(self, name, value):
-        return self.SpecialMethodName(self.context, name, value, self._builtin_context)
-
-
-def has_builtin_methods(cls):
-    base_dct = {}
-    # Need to care properly about inheritance. Builtin Methods should not get
-    # lost, just because they are not mentioned in a class.
-    for base_cls in reversed(cls.__bases__):
-        try:
-            base_dct.update(base_cls.builtin_methods)
-        except AttributeError:
-            pass
-
-    cls.builtin_methods = base_dct
-    for func in cls.__dict__.values():
-        try:
-            cls.builtin_methods.update(func.registered_builtin_methods)
-        except AttributeError:
-            pass
-    return cls
-
-
-def register_builtin_method(method_name, python_version_match=None):
-    def wrapper(func):
-        if python_version_match and python_version_match != 2 + int(is_py3):
-            # Some functions do only apply to certain versions.
-            return func
-        dct = func.__dict__.setdefault('registered_builtin_methods', {})
-        dct[method_name] = func
-        return func
-    return wrapper
-
-
-@has_builtin_methods
-class GeneratorMixin(object):
+class GeneratorBase(BuiltinOverwrite, IterableMixin):
     array_type = None
+    special_object_identifier = u'GENERATOR_OBJECT'
 
-    @register_builtin_method('send')
-    @register_builtin_method('next', python_version_match=2)
-    @register_builtin_method('__next__', python_version_match=3)
+    @publish_method('send')
+    @publish_method('next', python_version_match=2)
+    @publish_method('__next__', python_version_match=3)
     def py__next__(self):
-        # TODO add TypeError if params are given.
-        return unite(lazy_context.infer() for lazy_context in self.py__iter__())
-
-    def get_filters(self, search_global, until_position=None, origin_scope=None):
-        gen_obj = compiled.get_special_object(self.evaluator, 'GENERATOR_OBJECT')
-        yield SpecialMethodFilter(self, self.builtin_methods, gen_obj)
-        for filter in gen_obj.get_filters(search_global):
-            yield filter
-
-    def py__bool__(self):
-        return True
-
-    def py__class__(self):
-        gen_obj = compiled.get_special_object(self.evaluator, 'GENERATOR_OBJECT')
-        return gen_obj.py__class__()
+        return ContextSet.from_sets(lazy_context.infer() for lazy_context in self.py__iter__())
 
     @property
     def name(self):
         return compiled.CompiledContextName(self, 'generator')
 
 
-class Generator(GeneratorMixin, context.Context):
+class Generator(GeneratorBase):
     """Handling of `yield` functions."""
     def __init__(self, evaluator, func_execution_context):
-        super(Generator, self).__init__(evaluator, parent_context=evaluator.BUILTINS)
+        super(Generator, self).__init__(evaluator)
         self._func_execution_context = func_execution_context
 
     def py__iter__(self):
-        return self._func_execution_context.get_yield_values()
+        return self._func_execution_context.get_yield_lazy_contexts()
+
+    def py__stop_iteration_returns(self):
+        return self._func_execution_context.get_return_values()
 
     def __repr__(self):
         return "<%s of %s>" % (type(self).__name__, self._func_execution_context)
 
 
-class CompForContext(context.TreeContext):
+class CompForContext(TreeContext):
     @classmethod
     def from_comp_for(cls, parent_context, comp_for):
         return cls(parent_context.evaluator, parent_context, comp_for)
 
-    def __init__(self, evaluator, parent_context, comp_for):
-        super(CompForContext, self).__init__(evaluator, parent_context)
-        self.tree_node = comp_for
-
     def get_node(self):
         return self.tree_node
 
@@ -192,32 +90,33 @@ class CompForContext(context.TreeContext):
         yield ParserTreeFilter(self.evaluator, self)
 
 
-class Comprehension(AbstractSequence):
-    @staticmethod
-    def from_atom(evaluator, context, atom):
-        bracket = atom.children[0]
-        if bracket == '{':
-            if atom.children[1].children[1] == ':':
-                cls = DictComprehension
-            else:
-                cls = SetComprehension
-        elif bracket == '(':
-            cls = GeneratorComprehension
-        elif bracket == '[':
-            cls = ListComprehension
-        return cls(evaluator, context, atom)
+def comprehension_from_atom(evaluator, context, atom):
+    bracket = atom.children[0]
+    if bracket == '{':
+        if atom.children[1].children[1] == ':':
+            cls = DictComprehension
+        else:
+            cls = SetComprehension
+    elif bracket == '(':
+        cls = GeneratorComprehension
+    elif bracket == '[':
+        cls = ListComprehension
+    return cls(evaluator, context, atom)
 
+
+class ComprehensionMixin(object):
     def __init__(self, evaluator, defining_context, atom):
-        super(Comprehension, self).__init__(evaluator)
+        super(ComprehensionMixin, self).__init__(evaluator)
         self._defining_context = defining_context
         self._atom = atom
 
     def _get_comprehension(self):
+        "return 'a for a in b'"
         # The atom contains a testlist_comp
         return self._atom.children[1]
 
     def _get_comp_for(self):
-        # The atom contains a testlist_comp
+        "return CompFor('for a in b')"
         return self._get_comprehension().children[1]
 
     def _eval_node(self, index=0):
@@ -234,15 +133,18 @@ class Comprehension(AbstractSequence):
         return CompForContext.from_comp_for(parent_context, comp_for)
 
     def _nested(self, comp_fors, parent_context=None):
-        evaluator = self.evaluator
         comp_for = comp_fors[0]
-        input_node = comp_for.children[3]
+
+        is_async = 'async' == comp_for.children[comp_for.children.index('for') - 1]
+
+        input_node = comp_for.children[comp_for.children.index('in') + 1]
         parent_context = parent_context or self._defining_context
         input_types = parent_context.eval_node(input_node)
+        # TODO: simulate await if self.is_async
 
-        cn = context.ContextualizedNode(parent_context, input_node)
-        iterated = py__iter__(evaluator, input_types, cn)
-        exprlist = comp_for.children[1]
+        cn = ContextualizedNode(parent_context, input_node)
+        iterated = input_types.iterate(cn, is_async=is_async)
+        exprlist = comp_for.children[comp_for.children.index('for') + 1]
         for i, lazy_context in enumerate(iterated):
             types = lazy_context.infer()
             dct = unpack_tuple_to_dict(parent_context, types, exprlist)
@@ -250,7 +152,7 @@ class Comprehension(AbstractSequence):
                 parent_context,
                 comp_for,
             )
-            with helpers.predefine_names(context_, comp_for, dct):
+            with predefine_names(context_, comp_for, dct):
                 try:
                     for result in self._nested(comp_fors[1:], context_):
                         yield result
@@ -262,7 +164,7 @@ class Comprehension(AbstractSequence):
                         yield iterated
 
     @evaluator_method_cache(default=[])
-    @common.to_list
+    @to_list
     def _iterate(self):
         comp_fors = tuple(get_comp_fors(self._get_comp_for()))
         for result in self._nested(comp_fors):
@@ -270,20 +172,24 @@ class Comprehension(AbstractSequence):
 
     def py__iter__(self):
         for set_ in self._iterate():
-            yield context.LazyKnownContexts(set_)
+            yield LazyKnownContexts(set_)
 
     def __repr__(self):
         return "<%s of %s>" % (type(self).__name__, self._atom)
 
 
-class ArrayMixin(object):
-    def get_filters(self, search_global, until_position=None, origin_scope=None):
-        # `array.type` is a string with the type, e.g. 'list'.
+class Sequence(BuiltinOverwrite, IterableMixin):
+    api_type = u'instance'
+
+    @property
+    def name(self):
+        return compiled.CompiledContextName(self, self.array_type)
+
+    @memoize_method
+    def get_object(self):
         compiled_obj = compiled.builtin_from_name(self.evaluator, self.array_type)
-        yield SpecialMethodFilter(self, self.builtin_methods, compiled_obj)
-        for typ in compiled_obj.execute_evaluated(self):
-            for filter in typ.get_filters():
-                yield filter
+        only_obj, = compiled_obj.execute_evaluated(self)
+        return only_obj
 
     def py__bool__(self):
         return None  # We don't know the length, because of appends.
@@ -293,74 +199,88 @@ class ArrayMixin(object):
 
     @safe_property
     def parent(self):
-        return self.evaluator.BUILTINS
+        return self.evaluator.builtins_module
 
     def dict_values(self):
-        return unite(self._defining_context.eval_node(v) for k, v in self._items())
+        return ContextSet.from_sets(
+            self._defining_context.eval_node(v)
+            for k, v in self._items()
+        )
 
 
-class ListComprehension(ArrayMixin, Comprehension):
-    array_type = 'list'
+class ListComprehension(ComprehensionMixin, Sequence):
+    array_type = u'list'
 
     def py__getitem__(self, index):
         if isinstance(index, slice):
-            return set([self])
+            return ContextSet(self)
 
         all_types = list(self.py__iter__())
-        return all_types[index].infer()
+        with reraise_as_evaluator(IndexError, TypeError):
+            lazy_context = all_types[index]
+        return lazy_context.infer()
 
 
-class SetComprehension(ArrayMixin, Comprehension):
-    array_type = 'set'
+class SetComprehension(ComprehensionMixin, Sequence):
+    array_type = u'set'
 
 
-@has_builtin_methods
-class DictComprehension(ArrayMixin, Comprehension):
-    array_type = 'dict'
+class DictComprehension(ComprehensionMixin, Sequence):
+    array_type = u'dict'
 
     def _get_comp_for(self):
         return self._get_comprehension().children[3]
 
     def py__iter__(self):
         for keys, values in self._iterate():
-            yield context.LazyKnownContexts(keys)
+            yield LazyKnownContexts(keys)
 
     def py__getitem__(self, index):
         for keys, values in self._iterate():
             for k in keys:
                 if isinstance(k, compiled.CompiledObject):
-                    if k.obj == index:
+                    if k.get_safe_value(default=object()) == index:
                         return values
         return self.dict_values()
 
     def dict_values(self):
-        return unite(values for keys, values in self._iterate())
+        return ContextSet.from_sets(values for keys, values in self._iterate())
 
-    @register_builtin_method('values')
+    @publish_method('values')
     def _imitate_values(self):
-        lazy_context = context.LazyKnownContexts(self.dict_values())
-        return set([FakeSequence(self.evaluator, 'list', [lazy_context])])
+        lazy_context = LazyKnownContexts(self.dict_values())
+        return ContextSet(FakeSequence(self.evaluator, u'list', [lazy_context]))
 
-    @register_builtin_method('items')
+    @publish_method('items')
     def _imitate_items(self):
-        items = set(
-            FakeSequence(
-                self.evaluator, 'tuple'
-                (context.LazyKnownContexts(keys), context.LazyKnownContexts(values))
-            ) for keys, values in self._iterate()
-        )
+        lazy_contexts = [
+            LazyKnownContext(
+                FakeSequence(
+                    self.evaluator,
+                    u'tuple',
+                    [LazyKnownContexts(key),
+                     LazyKnownContexts(value)]
+                )
+            )
+            for key, value in self._iterate()
+        ]
 
-        return create_evaluated_sequence_set(self.evaluator, items, sequence_type='list')
+        return ContextSet(FakeSequence(self.evaluator, u'list', lazy_contexts))
+
+    def exact_key_items(self):
+        # NOTE: A smarter thing can probably done here to achieve better
+        # completions, but at least like this jedi doesn't crash
+        return []
 
 
-class GeneratorComprehension(GeneratorMixin, Comprehension):
+class GeneratorComprehension(ComprehensionMixin, GeneratorBase):
     pass
 
 
-class SequenceLiteralContext(ArrayMixin, AbstractSequence):
-    mapping = {'(': 'tuple',
-               '[': 'list',
-               '{': 'set'}
+class SequenceLiteralContext(Sequence):
+    mapping = {'(': u'tuple',
+               '[': u'list',
+               '{': u'set'}
 
     def __init__(self, evaluator, defining_context, atom):
         super(SequenceLiteralContext, self).__init__(evaluator)
@@ -368,52 +288,55 @@ class SequenceLiteralContext(ArrayMixin, AbstractSequence):
         self._defining_context = defining_context
 
         if self.atom.type in ('testlist_star_expr', 'testlist'):
-            self.array_type = 'tuple'
+            self.array_type = u'tuple'
         else:
             self.array_type = SequenceLiteralContext.mapping[atom.children[0]]
             """The builtin name of the array (list, set, tuple or dict)."""
 
     def py__getitem__(self, index):
         """Here the index is an int/str. Raises IndexError/KeyError."""
-        if self.array_type == 'dict':
+        if self.array_type == u'dict':
+            compiled_obj_index = compiled.create_simple_object(self.evaluator, index)
             for key, value in self._items():
                 for k in self._defining_context.eval_node(key):
                     if isinstance(k, compiled.CompiledObject) \
-                            and index == k.obj:
+                            and k.execute_operation(compiled_obj_index, u'==').get_safe_value():
                         return self._defining_context.eval_node(value)
-            raise KeyError('No key found in dictionary %s.' % self)
+            raise EvaluatorKeyError('No key found in dictionary %s.' % self)
 
         # Can raise an IndexError
         if isinstance(index, slice):
-            return set([self])
+            return ContextSet(self)
         else:
-            return self._defining_context.eval_node(self._items()[index])
+            with reraise_as_evaluator(TypeError, KeyError, IndexError):
+                node = self._items()[index]
+            return self._defining_context.eval_node(node)
 
     def py__iter__(self):
         """
         While values returns the possible values for any array field, this
         function returns the value for a certain index.
         """
-        if self.array_type == 'dict':
+        if self.array_type == u'dict':
             # Get keys.
-            types = set()
+            types = ContextSet()
             for k, _ in self._items():
                 types |= self._defining_context.eval_node(k)
             # We don't know which dict index comes first, therefore always
             # yield all the types.
             for _ in types:
-                yield context.LazyKnownContexts(types)
+                yield LazyKnownContexts(types)
         else:
             for node in self._items():
-                yield context.LazyTreeContext(self._defining_context, node)
+                yield LazyTreeContext(self._defining_context, node)
 
             for addition in check_array_additions(self._defining_context, self):
                 yield addition
 
     def _values(self):
         """Returns a list of a list of node."""
-        if self.array_type == 'dict':
-            return unite(v for k, v in self._items())
+        if self.array_type == u'dict':
+            return ContextSet.from_sets(v for k, v in self._items())
         else:
             return self._items()
 
@@ -428,21 +351,39 @@ class SequenceLiteralContext(ArrayMixin, AbstractSequence):
             return []  # Direct closing bracket, doesn't contain items.
 
         if array_node.type == 'testlist_comp':
-            return array_node.children[::2]
+            # filter out (for now) pep 448 single-star unpacking
+            return [value for value in array_node.children[::2]
+                    if value.type != "star_expr"]
         elif array_node.type == 'dictorsetmaker':
             kv = []
             iterator = iter(array_node.children)
             for key in iterator:
-                op = next(iterator, None)
-                if op is None or op == ',':
-                    kv.append(key)  # A set.
-                else:
-                    assert op == ':'  # A dict.
-                    kv.append((key, next(iterator)))
+                if key == "**":
+                    # dict with pep 448 double-star unpacking
+                    # for now ignoring the values imported by **
+                    next(iterator)
                     next(iterator, None)  # Possible comma.
+                else:
+                    op = next(iterator, None)
+                    if op is None or op == ',':
+                        if key.type == "star_expr":
+                            # pep 448 single-star unpacking
+                            # for now ignoring values imported by *
+                            pass
+                        else:
+                            kv.append(key)  # A set.
+                    else:
+                        assert op == ':'  # A dict.
+                        kv.append((key, next(iterator)))
+                        next(iterator, None)  # Possible comma.
             return kv
         else:
-            return [array_node]
+            if array_node.type == "star_expr":
+                # pep 448 single-star unpacking
+                # for now ignoring values imported by *
+                return []
+            else:
+                return [array_node]
 
     def exact_key_items(self):
         """
@@ -451,38 +392,37 @@ class SequenceLiteralContext(ArrayMixin, AbstractSequence):
         """
         for key_node, value in self._items():
             for key in self._defining_context.eval_node(key_node):
-                if precedence.is_string(key):
-                    yield key.obj, context.LazyTreeContext(self._defining_context, value)
+                if is_string(key):
+                    yield key.get_safe_value(), LazyTreeContext(self._defining_context, value)
 
     def __repr__(self):
         return "<%s of %s>" % (self.__class__.__name__, self.atom)
 
 
-@has_builtin_methods
 class DictLiteralContext(SequenceLiteralContext):
-    array_type = 'dict'
+    array_type = u'dict'
 
     def __init__(self, evaluator, defining_context, atom):
         super(SequenceLiteralContext, self).__init__(evaluator)
         self._defining_context = defining_context
         self.atom = atom
 
-    @register_builtin_method('values')
+    @publish_method('values')
     def _imitate_values(self):
-        lazy_context = context.LazyKnownContexts(self.dict_values())
-        return set([FakeSequence(self.evaluator, 'list', [lazy_context])])
+        lazy_context = LazyKnownContexts(self.dict_values())
+        return ContextSet(FakeSequence(self.evaluator, u'list', [lazy_context]))
 
-    @register_builtin_method('items')
+    @publish_method('items')
     def _imitate_items(self):
         lazy_contexts = [
-            context.LazyKnownContext(FakeSequence(
-                self.evaluator, 'tuple',
-                (context.LazyTreeContext(self._defining_context, key_node),
-                 context.LazyTreeContext(self._defining_context, value_node))
+            LazyKnownContext(FakeSequence(
+                self.evaluator, u'tuple',
+                (LazyTreeContext(self._defining_context, key_node),
+                 LazyTreeContext(self._defining_context, value_node))
             )) for key_node, value_node in self._items()
         ]
 
-        return set([FakeSequence(self.evaluator, 'list', lazy_contexts)])
+        return ContextSet(FakeSequence(self.evaluator, u'list', lazy_contexts))
 
 
 class _FakeArray(SequenceLiteralContext):
@@ -502,7 +442,9 @@ class FakeSequence(_FakeArray):
         self._lazy_context_list = lazy_context_list
 
     def py__getitem__(self, index):
-        return set(self._lazy_context_list[index].infer())
+        with reraise_as_evaluator(IndexError, TypeError):
+            lazy_context = self._lazy_context_list[index]
+        return lazy_context.infer()
 
     def py__iter__(self):
         return self._lazy_context_list
@@ -516,18 +458,42 @@ class FakeSequence(_FakeArray):
 
 class FakeDict(_FakeArray):
     def __init__(self, evaluator, dct):
-        super(FakeDict, self).__init__(evaluator, dct, 'dict')
+        super(FakeDict, self).__init__(evaluator, dct, u'dict')
         self._dct = dct
 
     def py__iter__(self):
         for key in self._dct:
-            yield context.LazyKnownContext(compiled.create(self.evaluator, key))
+            yield LazyKnownContext(compiled.create_simple_object(self.evaluator, key))
 
     def py__getitem__(self, index):
-        return self._dct[index].infer()
+        if is_py3 and self.evaluator.environment.version_info.major == 2:
+            # In Python 2 bytes and unicode compare.
+            if isinstance(index, bytes):
+                index_unicode = force_unicode(index)
+                try:
+                    return self._dct[index_unicode].infer()
+                except KeyError:
+                    pass
+            elif isinstance(index, str):
+                index_bytes = index.encode('utf-8')
+                try:
+                    return self._dct[index_bytes].infer()
+                except KeyError:
+                    pass
+
+        with reraise_as_evaluator(KeyError):
+            lazy_context = self._dct[index]
+        return lazy_context.infer()
+
+    @publish_method('values')
+    def _values(self):
+        return ContextSet(FakeSequence(
+            self.evaluator, u'tuple',
+            [LazyKnownContexts(self.dict_values())]
+        ))
 
     def dict_values(self):
-        return unite(lazy_context.infer() for lazy_context in self._dct.values())
+        return ContextSet.from_sets(lazy_context.infer() for lazy_context in self._dct.values())
 
     def exact_key_items(self):
         return self._dct.items()
@@ -544,7 +510,7 @@ class MergedArray(_FakeArray):
                 yield lazy_context
 
     def py__getitem__(self, index):
-        return unite(lazy_context.infer() for lazy_context in self.py__iter__())
+        return ContextSet.from_sets(lazy_context.infer() for lazy_context in self.py__iter__())
 
     def _items(self):
         for array in self._arrays:
@@ -568,7 +534,7 @@ def unpack_tuple_to_dict(context, types, exprlist):
         dct = {}
         parts = iter(exprlist.children[::2])
         n = 0
-        for lazy_context in py__iter__(context.evaluator, types, exprlist):
+        for lazy_context in types.iterate(exprlist):
             n += 1
             try:
                 part = next(parts)
@@ -595,103 +561,16 @@ def unpack_tuple_to_dict(context, types, exprlist):
     raise NotImplementedError
 
 
-def py__iter__(evaluator, types, contextualized_node=None):
-    debug.dbg('py__iter__')
-    type_iters = []
-    for typ in types:
-        try:
-            iter_method = typ.py__iter__
-        except AttributeError:
-            if contextualized_node is not None:
-                analysis.add(
-                    contextualized_node.context,
-                    'type-error-not-iterable',
-                    contextualized_node._node,
-                    message="TypeError: '%s' object is not iterable" % typ)
-        else:
-            type_iters.append(iter_method())
-
-    for lazy_contexts in zip_longest(*type_iters):
-        yield context.get_merged_lazy_context(
-            [l for l in lazy_contexts if l is not None]
-        )
-
-
-def py__iter__types(evaluator, types, contextualized_node=None):
-    """
-    Calls `py__iter__`, but ignores the ordering in the end and just returns
-    all types that it contains.
-    """
-    return unite(
-        lazy_context.infer()
-        for lazy_context in py__iter__(evaluator, types, contextualized_node)
-    )
-
-
-def py__getitem__(evaluator, context, types, trailer):
-    from jedi.evaluate.representation import ClassContext
-    from jedi.evaluate.instance import TreeInstance
-    result = set()
-
-    trailer_op, node, trailer_cl = trailer.children
-    assert trailer_op == "["
-    assert trailer_cl == "]"
-
-    # special case: PEP0484 typing module, see
-    # https://github.com/davidhalter/jedi/issues/663
-    for typ in list(types):
-        if isinstance(typ, (ClassContext, TreeInstance)):
-            typing_module_types = pep0484.py__getitem__(context, typ, node)
-            if typing_module_types is not None:
-                types.remove(typ)
-                result |= typing_module_types
-
-    if not types:
-        # all consumed by special cases
-        return result
-
-    for index in create_index_types(evaluator, context, node):
-        if isinstance(index, (compiled.CompiledObject, Slice)):
-            index = index.obj
-
-        if type(index) not in (float, int, str, unicode, slice, type(Ellipsis)):
-            # If the index is not clearly defined, we have to get all the
-            # possiblities.
-            for typ in list(types):
-                if isinstance(typ, AbstractSequence) and typ.array_type == 'dict':
-                    types.remove(typ)
-                    result |= typ.dict_values()
-            return result | py__iter__types(evaluator, types)
-
-        for typ in types:
-            # The actual getitem call.
-            try:
-                getitem = typ.py__getitem__
-            except AttributeError:
-                # TODO this context is probably not right.
-                analysis.add(context, 'type-error-not-subscriptable', trailer_op,
-                             message="TypeError: '%s' object is not subscriptable" % typ)
-            else:
-                try:
-                    result |= getitem(index)
-                except IndexError:
-                    result |= py__iter__types(evaluator, set([typ]))
-                except KeyError:
-                    # Must be a dict. Lists don't raise KeyErrors.
-                    result |= typ.dict_values()
-    return result
-
-
 def check_array_additions(context, sequence):
     """ Just a mapper function for the internal _check_array_additions """
     if sequence.array_type not in ('list', 'set'):
         # TODO also check for dict updates
-        return set()
+        return NO_CONTEXTS
 
     return _check_array_additions(context, sequence)
 
 
-@evaluator_method_cache(default=set())
+@evaluator_method_cache(default=NO_CONTEXTS)
 @debug.increase_indent
 def _check_array_additions(context, sequence):
     """
@@ -700,25 +579,25 @@ def _check_array_additions(context, sequence):
     >>> a = [""]
     >>> a.append(1)
     """
-    from jedi.evaluate import param
+    from jedi.evaluate import arguments
 
     debug.dbg('Dynamic array search for %s' % sequence, color='MAGENTA')
     module_context = context.get_root_context()
     if not settings.dynamic_array_additions or isinstance(module_context, compiled.CompiledObject):
         debug.dbg('Dynamic array search aborted.', color='MAGENTA')
-        return set()
+        return ContextSet()
 
     def find_additions(context, arglist, add_name):
-        params = list(param.TreeArguments(context.evaluator, context, arglist).unpack())
+        params = list(arguments.TreeArguments(context.evaluator, context, arglist).unpack())
         result = set()
         if add_name in ['insert']:
             params = params[1:]
         if add_name in ['append', 'add', 'insert']:
-            for key, lazy_context in params:
-                result.add(lazy_context)
+            for key, whatever in params:
+                result.add(whatever)
         elif add_name in ['extend', 'update']:
             for key, lazy_context in params:
-                result |= set(py__iter__(context.evaluator, lazy_context.infer()))
+                result |= set(lazy_context.infer().iterate())
         return result
 
     temp_param_add, settings.dynamic_params_for_other_modules = \
@@ -755,7 +634,7 @@ def _check_array_additions(context, sequence):
 
                 with recursion.execution_allowed(context.evaluator, power) as allowed:
                     if allowed:
-                        found = helpers.evaluate_call_of_leaf(
+                        found = evaluate_call_of_leaf(
                             random_context,
                             name,
                             cut_own_trailer=True
@@ -774,14 +653,11 @@ def _check_array_additions(context, sequence):
     return added_types
 
 
-def get_dynamic_array_instance(instance):
+def get_dynamic_array_instance(instance, arguments):
     """Used for set() and list() instances."""
-    if not settings.dynamic_array_additions:
-        return instance.var_args
-
-    ai = _ArrayInstance(instance)
-    from jedi.evaluate import param
-    return param.ValuesArguments([[ai]])
+    ai = _ArrayInstance(instance, arguments)
+    from jedi.evaluate import arguments
+    return arguments.ValuesArguments([ContextSet(ai)])
 
 
 class _ArrayInstance(object):
@@ -795,9 +671,9 @@ class _ArrayInstance(object):
     and therefore doesn't need filters, `py__bool__` and so on, because
     we don't use these operations in `builtins.py`.
     """
-    def __init__(self, instance):
+    def __init__(self, instance, var_args):
         self.instance = instance
-        self.var_args = instance.var_args
+        self.var_args = var_args
 
     def py__iter__(self):
         var_args = self.var_args
@@ -806,21 +682,24 @@ class _ArrayInstance(object):
         except StopIteration:
             pass
         else:
-            for lazy in py__iter__(self.instance.evaluator, lazy_context.infer()):
+            for lazy in lazy_context.infer().iterate():
                 yield lazy
 
-        from jedi.evaluate import param
-        if isinstance(var_args, param.TreeArguments):
+        from jedi.evaluate import arguments
+        if isinstance(var_args, arguments.TreeArguments):
             additions = _check_array_additions(var_args.context, self.instance)
             for addition in additions:
                 yield addition
 
+    def iterate(self, contextualized_node=None, is_async=False):
+        return self.py__iter__()
 
-class Slice(context.Context):
+
+class Slice(Context):
     def __init__(self, context, start, stop, step):
         super(Slice, self).__init__(
             context.evaluator,
-            parent_context=context.evaluator.BUILTINS
+            parent_context=context.evaluator.builtins_module
         )
         self._context = context
         # all of them are either a Precedence or None.
@@ -843,42 +722,11 @@ class Slice(context.Context):
                 # For simplicity, we want slices to be clear defined with just
                 # one type.  Otherwise we will return an empty slice object.
                 raise IndexError
-            try:
-                return list(result)[0].obj
-            except AttributeError:
-                return None
+
+            context, = result
+            return get_int_or_none(context)
 
         try:
             return slice(get(self._start), get(self._stop), get(self._step))
         except IndexError:
             return slice(None, None, None)
-
-
-def create_index_types(evaluator, context, index):
-    """
-    Handles slices in subscript nodes.
-    """
-    if index == ':':
-        # Like array[:]
-        return set([Slice(context, None, None, None)])
-
-    elif index.type == 'subscript' and not index.children[0] == '.':
-        # subscript basically implies a slice operation, except for Python 2's
-        # Ellipsis.
-        # e.g. array[:3]
-        result = []
-        for el in index.children:
-            if el == ':':
-                if not result:
-                    result.append(None)
-            elif el.type == 'sliceop':
-                if len(el.children) == 2:
-                    result.append(el.children[1])
-            else:
-                result.append(el)
-        result += [None] * (3 - len(result))
-
-        return set([Slice(context, *result)])
-
-    # No slices
-    return context.eval_node(index)

jedi/evaluate/context/klass.py

@@ -0,0 +1,221 @@
+"""
+Like described in the :mod:`parso.python.tree` module,
+there's a need for an ast like module to represent the states of parsed
+modules.
+
+But now there are also structures in Python that need a little bit more than
+that. An ``Instance`` for example is only a ``Class`` before it is
+instantiated. This class represents these cases.
+
+So, why is there also a ``Class`` class here? Well, there are decorators and
+they change classes in Python 3.
+
+Representation modules also define "magic methods". Those methods look like
+``py__foo__`` and are typically mappable to the Python equivalents ``__call__``
+and others. Here's a list:
+
+====================================== ========================================
+**Method**                             **Description**
+-------------------------------------- ----------------------------------------
+py__call__(params: Array)              On callable objects, returns types.
+py__bool__()                           Returns True/False/None; None means that
+                                       there's no certainty.
+py__bases__()                          Returns a list of base classes.
+py__mro__()                            Returns a list of classes (the mro).
+py__iter__()                           Returns a generator of a set of types.
+py__class__()                          Returns the class of an instance.
+py__getitem__(index: int/str)          Returns a a set of types of the index.
+                                       Can raise an IndexError/KeyError.
+py__file__()                           Only on modules. Returns None if does
+                                       not exist.
+py__package__()                        Only on modules. For the import system.
+py__path__()                           Only on modules. For the import system.
+py__get__(call_object)                 Only on instances. Simulates
+                                       descriptors.
+py__doc__(include_call_signature:      Returns the docstring for a context.
+          bool)
+====================================== ========================================
+
+"""
+from jedi._compatibility import use_metaclass
+from jedi.parser_utils import get_parent_scope
+from jedi.evaluate.cache import evaluator_method_cache, CachedMetaClass
+from jedi.evaluate import compiled
+from jedi.evaluate.lazy_context import LazyKnownContext
+from jedi.evaluate.filters import ParserTreeFilter, TreeNameDefinition, \
+    ContextName
+from jedi.evaluate.base_context import ContextSet, iterator_to_context_set, \
+    TreeContext
+
+
+def apply_py__get__(context, base_context):
+    try:
+        method = context.py__get__
+    except AttributeError:
+        yield context
+    else:
+        for descriptor_context in method(base_context):
+            yield descriptor_context
+
+
+class ClassName(TreeNameDefinition):
+    def __init__(self, parent_context, tree_name, name_context, apply_decorators):
+        super(ClassName, self).__init__(parent_context, tree_name)
+        self._name_context = name_context
+        self._apply_decorators = apply_decorators
+
+    @iterator_to_context_set
+    def infer(self):
+        # TODO this _name_to_types might get refactored and be a part of the
+        # parent class. Once it is, we can probably just overwrite method to
+        # achieve this.
+        from jedi.evaluate.syntax_tree import tree_name_to_contexts
+        inferred = tree_name_to_contexts(
+            self.parent_context.evaluator, self._name_context, self.tree_name)
+
+        for result_context in inferred:
+            if self._apply_decorators:
+                for c in apply_py__get__(result_context, self.parent_context):
+                    yield c
+            else:
+                yield result_context
+
+
+class ClassFilter(ParserTreeFilter):
+    name_class = ClassName
+
+    def __init__(self, *args, **kwargs):
+        self._is_instance = kwargs.pop('is_instance')  # Python 2 :/
+        super(ClassFilter, self).__init__(*args, **kwargs)
+
+    def _convert_names(self, names):
+        return [
+            self.name_class(
+                parent_context=self.context,
+                tree_name=name,
+                name_context=self._node_context,
+                apply_decorators=not self._is_instance,
+            ) for name in names
+        ]
+
+    def _equals_origin_scope(self):
+        node = self._origin_scope
+        while node is not None:
+            if node == self._parser_scope or node == self.context:
+                return True
+            node = get_parent_scope(node)
+        return False
+
+    def _access_possible(self, name):
+        return not name.value.startswith('__') or name.value.endswith('__') \
+            or self._equals_origin_scope()
+
+    def _filter(self, names):
+        names = super(ClassFilter, self)._filter(names)
+        return [name for name in names if self._access_possible(name)]
+
+
+class ClassContext(use_metaclass(CachedMetaClass, TreeContext)):
+    """
+    This class is not only important to extend `tree.Class`, it is also a
+    important for descriptors (if the descriptor methods are evaluated or not).
+    """
+    api_type = u'class'
+
+    @evaluator_method_cache(default=())
+    def py__mro__(self):
+        def add(cls):
+            if cls not in mro:
+                mro.append(cls)
+
+        mro = [self]
+        # TODO Do a proper mro resolution. Currently we are just listing
+        # classes. However, it's a complicated algorithm.
+        for lazy_cls in self.py__bases__():
+            # TODO there's multiple different mro paths possible if this yields
+            # multiple possibilities. Could be changed to be more correct.
+            for cls in lazy_cls.infer():
+                # TODO detect for TypeError: duplicate base class str,
+                # e.g.  `class X(str, str): pass`
+                try:
+                    mro_method = cls.py__mro__
+                except AttributeError:
+                    # TODO add a TypeError like:
+                    """
+                    >>> class Y(lambda: test): pass
+                    Traceback (most recent call last):
+                      File "<stdin>", line 1, in <module>
+                    TypeError: function() argument 1 must be code, not str
+                    >>> class Y(1): pass
+                    Traceback (most recent call last):
+                      File "<stdin>", line 1, in <module>
+                    TypeError: int() takes at most 2 arguments (3 given)
+                    """
+                    pass
+                else:
+                    add(cls)
+                    for cls_new in mro_method():
+                        add(cls_new)
+        return tuple(mro)
+
+    @evaluator_method_cache(default=())
+    def py__bases__(self):
+        arglist = self.tree_node.get_super_arglist()
+        if arglist:
+            from jedi.evaluate import arguments
+            args = arguments.TreeArguments(self.evaluator, self.parent_context, arglist)
+            return [value for key, value in args.unpack() if key is None]
+        else:
+            return [LazyKnownContext(compiled.builtin_from_name(self.evaluator, u'object'))]
+
+    def py__call__(self, params):
+        from jedi.evaluate.context import TreeInstance
+        return ContextSet(TreeInstance(self.evaluator, self.parent_context, self, params))
+
+    def py__class__(self):
+        return compiled.builtin_from_name(self.evaluator, u'type')
+
+    def get_filters(self, search_global, until_position=None, origin_scope=None, is_instance=False):
+        if search_global:
+            yield ParserTreeFilter(
+                self.evaluator,
+                context=self,
+                until_position=until_position,
+                origin_scope=origin_scope
+            )
+        else:
+            for cls in self.py__mro__():
+                if isinstance(cls, compiled.CompiledObject):
+                    for filter in cls.get_filters(is_instance=is_instance):
+                        yield filter
+                else:
+                    yield ClassFilter(
+                        self.evaluator, self, node_context=cls,
+                        origin_scope=origin_scope,
+                        is_instance=is_instance
+                    )
+
+    def is_class(self):
+        return True
+
+    def get_function_slot_names(self, name):
+        for filter in self.get_filters(search_global=False):
+            names = filter.get(name)
+            if names:
+                return names
+        return []
+
+    def get_param_names(self):
+        for name in self.get_function_slot_names(u'__init__'):
+            for context_ in name.infer():
+                try:
+                    method = context_.get_param_names
+                except AttributeError:
+                    pass
+                else:
+                    return list(method())[1:]
+        return []
+
+    @property
+    def name(self):
+        return ContextName(self, self.tree_node.name)

jedi/evaluate/context/module.py

@@ -0,0 +1,219 @@
+import re
+import os
+
+from parso import python_bytes_to_unicode
+
+from jedi.evaluate.cache import evaluator_method_cache
+from jedi._compatibility import iter_modules, all_suffixes
+from jedi.evaluate.filters import GlobalNameFilter, ContextNameMixin, \
+    AbstractNameDefinition, ParserTreeFilter, DictFilter, MergedFilter
+from jedi.evaluate import compiled
+from jedi.evaluate.base_context import TreeContext
+from jedi.evaluate.imports import SubModuleName, infer_import
+
+
+class _ModuleAttributeName(AbstractNameDefinition):
+    """
+    For module attributes like __file__, __str__ and so on.
+    """
+    api_type = u'instance'
+
+    def __init__(self, parent_module, string_name):
+        self.parent_context = parent_module
+        self.string_name = string_name
+
+    def infer(self):
+        return compiled.get_string_context_set(self.parent_context.evaluator)
+
+
+class ModuleName(ContextNameMixin, AbstractNameDefinition):
+    start_pos = 1, 0
+
+    def __init__(self, context, name):
+        self._context = context
+        self._name = name
+
+    @property
+    def string_name(self):
+        return self._name
+
+
+class ModuleContext(TreeContext):
+    api_type = u'module'
+    parent_context = None
+
+    def __init__(self, evaluator, module_node, path, code_lines):
+        super(ModuleContext, self).__init__(
+            evaluator,
+            parent_context=None,
+            tree_node=module_node
+        )
+        self._path = path
+        self.code_lines = code_lines
+
+    def get_filters(self, search_global, until_position=None, origin_scope=None):
+        yield MergedFilter(
+            ParserTreeFilter(
+                self.evaluator,
+                context=self,
+                until_position=until_position,
+                origin_scope=origin_scope
+            ),
+            GlobalNameFilter(self, self.tree_node),
+        )
+        yield DictFilter(self._sub_modules_dict())
+        yield DictFilter(self._module_attributes_dict())
+        for star_module in self.star_imports():
+            yield next(star_module.get_filters(search_global))
+
+    # I'm not sure if the star import cache is really that effective anymore
+    # with all the other really fast import caches. Recheck. Also we would need
+    # to push the star imports into Evaluator.module_cache, if we reenable this.
+    @evaluator_method_cache([])
+    def star_imports(self):
+        modules = []
+        for i in self.tree_node.iter_imports():
+            if i.is_star_import():
+                name = i.get_paths()[-1][-1]
+                new = infer_import(self, name)
+                for module in new:
+                    if isinstance(module, ModuleContext):
+                        modules += module.star_imports()
+                modules += new
+        return modules
+
+    @evaluator_method_cache()
+    def _module_attributes_dict(self):
+        names = ['__file__', '__package__', '__doc__', '__name__']
+        # All the additional module attributes are strings.
+        return dict((n, _ModuleAttributeName(self, n)) for n in names)
+
+    @property
+    def _string_name(self):
+        """ This is used for the goto functions. """
+        if self._path is None:
+            return ''  # no path -> empty name
+        else:
+            sep = (re.escape(os.path.sep),) * 2
+            r = re.search(r'([^%s]*?)(%s__init__)?(\.py|\.so)?$' % sep, self._path)
+            # Remove PEP 3149 names
+            return re.sub(r'\.[a-z]+-\d{2}[mud]{0,3}$', '', r.group(1))
+
+    @property
+    @evaluator_method_cache()
+    def name(self):
+        return ModuleName(self, self._string_name)
+
+    def _get_init_directory(self):
+        """
+        :return: The path to the directory of a package. None in case it's not
+                 a package.
+        """
+        for suffix in all_suffixes():
+            ending = '__init__' + suffix
+            py__file__ = self.py__file__()
+            if py__file__ is not None and py__file__.endswith(ending):
+                # Remove the ending, including the separator.
+                return self.py__file__()[:-len(ending) - 1]
+        return None
+
+    def py__name__(self):
+        for name, module in self.evaluator.module_cache.iterate_modules_with_names():
+            if module == self and name != '':
+                return name
+
+        return '__main__'
+
+    def py__file__(self):
+        """
+        In contrast to Python's __file__ can be None.
+        """
+        if self._path is None:
+            return None
+
+        return os.path.abspath(self._path)
+
+    def py__package__(self):
+        if self._get_init_directory() is None:
+            return re.sub(r'\.?[^.]+$', '', self.py__name__())
+        else:
+            return self.py__name__()
+
+    def _py__path__(self):
+        search_path = self.evaluator.get_sys_path()
+        init_path = self.py__file__()
+        if os.path.basename(init_path) == '__init__.py':
+            with open(init_path, 'rb') as f:
+                content = python_bytes_to_unicode(f.read(), errors='replace')
+                # these are strings that need to be used for namespace packages,
+                # the first one is ``pkgutil``, the second ``pkg_resources``.
+                options = ('declare_namespace(__name__)', 'extend_path(__path__')
+                if options[0] in content or options[1] in content:
+                    # It is a namespace, now try to find the rest of the
+                    # modules on sys_path or whatever the search_path is.
+                    paths = set()
+                    for s in search_path:
+                        other = os.path.join(s, self.name.string_name)
+                        if os.path.isdir(other):
+                            paths.add(other)
+                    if paths:
+                        return list(paths)
+                    # TODO I'm not sure if this is how nested namespace
+                    # packages work. The tests are not really good enough to
+                    # show that.
+        # Default to this.
+        return [self._get_init_directory()]
+
+    @property
+    def py__path__(self):
+        """
+        Not seen here, since it's a property. The callback actually uses a
+        variable, so use it like::
+
+            foo.py__path__(sys_path)
+
+        In case of a package, this returns Python's __path__ attribute, which
+        is a list of paths (strings).
+        Raises an AttributeError if the module is not a package.
+        """
+        path = self._get_init_directory()
+
+        if path is None:
+            raise AttributeError('Only packages have __path__ attributes.')
+        else:
+            return self._py__path__
+
+    @evaluator_method_cache()
+    def _sub_modules_dict(self):
+        """
+        Lists modules in the directory of this module (if this module is a
+        package).
+        """
+        names = {}
+        try:
+            method = self.py__path__
+        except AttributeError:
+            pass
+        else:
+            for path in method():
+                mods = iter_modules([path])
+                for module_loader, name, is_pkg in mods:
+                    # It's obviously a relative import to the current module.
+                    names[name] = SubModuleName(self, name)
+
+        # TODO add something like this in the future, its cleaner than the
+        #   import hacks.
+        # ``os.path`` is a hardcoded exception, because it's a
+        # ``sys.modules`` modification.
+        # if str(self.name) == 'os':
+        #     names.append(Name('path', parent_context=self))
+
+        return names
+
+    def py__class__(self):
+        return compiled.get_special_object(self.evaluator, u'MODULE_CLASS')
+
+    def __repr__(self):
+        return "<%s: %s@%s-%s>" % (
+            self.__class__.__name__, self._string_name,
+            self.tree_node.start_pos[0], self.tree_node.end_pos[0])

jedi/evaluate/context/namespace.py

@@ -0,0 +1,72 @@
+import os
+from itertools import chain
+
+from jedi.evaluate.cache import evaluator_method_cache
+from jedi.evaluate import imports
+from jedi.evaluate.filters import DictFilter, AbstractNameDefinition, ContextNameMixin
+from jedi.evaluate.base_context import Context
+
+
+class ImplicitNSName(ContextNameMixin, AbstractNameDefinition):
+    """
+    Accessing names for implicit namespace packages should infer to nothing.
+    This object will prevent Jedi from raising exceptions
+    """
+    def __init__(self, implicit_ns_context, string_name):
+        self._context = implicit_ns_context
+        self.string_name = string_name
+
+
+class ImplicitNamespaceContext(Context):
+    """
+    Provides support for implicit namespace packages
+    """
+    # Is a module like every other module, because if you import an empty
+    # folder foobar it will be available as an object:
+    # <module 'foobar' (namespace)>.
+    api_type = u'module'
+    parent_context = None
+
+    def __init__(self, evaluator, fullname, paths):
+        super(ImplicitNamespaceContext, self).__init__(evaluator, parent_context=None)
+        self.evaluator = evaluator
+        self._fullname = fullname
+        self.paths = paths
+
+    def get_filters(self, search_global, until_position=None, origin_scope=None):
+        yield DictFilter(self._sub_modules_dict())
+
+    @property
+    @evaluator_method_cache()
+    def name(self):
+        string_name = self.py__package__().rpartition('.')[-1]
+        return ImplicitNSName(self, string_name)
+
+    def py__file__(self):
+        return None
+
+    def py__package__(self):
+        """Return the fullname
+        """
+        return self._fullname
+
+    def py__path__(self):
+        return [self.paths]
+
+    def py__name__(self):
+        return self._fullname
+
+    @evaluator_method_cache()
+    def _sub_modules_dict(self):
+        names = {}
+
+        file_names = chain.from_iterable(os.listdir(path) for path in self.paths)
+        mods = [
+            file_name.rpartition('.')[0] if '.' in file_name else file_name
+            for file_name in file_names
+            if file_name != '__pycache__'
+        ]
+
+        for name in mods:
+            names[name] = imports.SubModuleName(self, name)
+        return names

jedi/evaluate/docstrings.py

@@ -18,14 +18,14 @@ annotations.
 import re
 from textwrap import dedent
 
-from parso import parse
+from parso import parse, ParserSyntaxError
 
 from jedi._compatibility import u
-from jedi.common import unite
-from jedi.evaluate import context
+from jedi.evaluate.utils import indent_block
 from jedi.evaluate.cache import evaluator_method_cache
-from jedi.common import indent_block
-from jedi.evaluate.iterable import SequenceLiteralContext, FakeSequence
+from jedi.evaluate.base_context import iterator_to_context_set, ContextSet, \
+    NO_CONTEXTS
+from jedi.evaluate.lazy_context import LazyKnownContexts
 
 
 DOCSTRING_PARAM_PATTERNS = [
@@ -42,49 +42,60 @@ DOCSTRING_RETURN_PATTERNS = [
 REST_ROLE_PATTERN = re.compile(r':[^`]+:`([^`]+)`')
 
 
-try:
-    from numpydoc.docscrape import NumpyDocString
-except ImportError:
-    def _search_param_in_numpydocstr(docstr, param_str):
-        return []
+_numpy_doc_string_cache = None
 
-    def _search_return_in_numpydocstr(docstr):
-        return []
-else:
-    def _search_param_in_numpydocstr(docstr, param_str):
-        """Search `docstr` (in numpydoc format) for type(-s) of `param_str`."""
-        try:
-            # This is a non-public API. If it ever changes we should be 
-            # prepared and return gracefully.
-            params = NumpyDocString(docstr)._parsed_data['Parameters']
-        except (KeyError, AttributeError):
-            return []
-        for p_name, p_type, p_descr in params:
-            if p_name == param_str:
-                m = re.match('([^,]+(,[^,]+)*?)(,[ ]*optional)?$', p_type)
-                if m:
-                    p_type = m.group(1)
-                return list(_expand_typestr(p_type))
-        return []
 
-    def _search_return_in_numpydocstr(docstr):
-        """
-        Search `docstr` (in numpydoc format) for type(-s) of function returns.
-        """
-        doc = NumpyDocString(docstr)
-        try:
-            # This is a non-public API. If it ever changes we should be 
-            # prepared and return gracefully.
-            returns = doc._parsed_data['Returns']
-            returns += doc._parsed_data['Yields']
-        except (KeyError, AttributeError):
-            raise StopIteration
-        for r_name, r_type, r_descr in returns:
-            #Return names are optional and if so the type is in the name
-            if not r_type:
-                r_type = r_name
-            for type_ in _expand_typestr(r_type):
-                yield type_
+def _get_numpy_doc_string_cls():
+    global _numpy_doc_string_cache
+    if isinstance(_numpy_doc_string_cache, ImportError):
+        raise _numpy_doc_string_cache
+    try:
+        from numpydoc.docscrape import NumpyDocString
+        _numpy_doc_string_cache = NumpyDocString
+    except ImportError as e:
+        _numpy_doc_string_cache = e
+        raise
+    return _numpy_doc_string_cache
+
+
+def _search_param_in_numpydocstr(docstr, param_str):
+    """Search `docstr` (in numpydoc format) for type(-s) of `param_str`."""
+    try:
+        # This is a non-public API. If it ever changes we should be
+        # prepared and return gracefully.
+        params = _get_numpy_doc_string_cls()(docstr)._parsed_data['Parameters']
+    except (KeyError, AttributeError, ImportError):
+        return []
+    for p_name, p_type, p_descr in params:
+        if p_name == param_str:
+            m = re.match(r'([^,]+(,[^,]+)*?)(,[ ]*optional)?$', p_type)
+            if m:
+                p_type = m.group(1)
+            return list(_expand_typestr(p_type))
+    return []
+
+
+def _search_return_in_numpydocstr(docstr):
+    """
+    Search `docstr` (in numpydoc format) for type(-s) of function returns.
+    """
+    try:
+        doc = _get_numpy_doc_string_cls()(docstr)
+    except ImportError:
+        return
+    try:
+        # This is a non-public API. If it ever changes we should be
+        # prepared and return gracefully.
+        returns = doc._parsed_data['Returns']
+        returns += doc._parsed_data['Yields']
+    except (KeyError, AttributeError):
+        return
+    for r_name, r_type, r_descr in returns:
+        # Return names are optional and if so the type is in the name
+        if not r_type:
+            r_type = r_name
+        for type_ in _expand_typestr(r_type):
+            yield type_
 
 
 def _expand_typestr(type_str):
@@ -92,11 +103,11 @@ def _expand_typestr(type_str):
     Attempts to interpret the possible types in `type_str`
     """
     # Check if alternative types are specified with 'or'
-    if re.search('\\bor\\b', type_str):
+    if re.search(r'\bor\b', type_str):
         for t in type_str.split('or'):
             yield t.split('of')[0].strip()
     # Check if like "list of `type`" and set type to list
-    elif re.search('\\bof\\b', type_str):
+    elif re.search(r'\bof\b', type_str):
         yield type_str.split('of')[0]
     # Check if type has is a set of valid literal values eg: {'C', 'F', 'A'}
     elif type_str.startswith('{'):
@@ -145,8 +156,7 @@ def _search_param_in_docstr(docstr, param_str):
         if match:
             return [_strip_rst_role(match.group(1))]
 
-    return (_search_param_in_numpydocstr(docstr, param_str) or
-            [])
+    return _search_param_in_numpydocstr(docstr, param_str)
 
 
 def _strip_rst_role(type_str):
@@ -179,12 +189,12 @@ def _evaluate_for_statement_string(module_context, string):
         Need this docstring so that if the below part is not valid Python this
         is still a function.
         '''
-    {0}
+    {}
     """))
     if string is None:
         return []
 
-    for element in re.findall('((?:\w+\.)*\w+)\.', string):
+    for element in re.findall(r'((?:\w+\.)*\w+)\.', string):
         # Try to import module part in dotted name.
         # (e.g., 'threading' in 'threading.Thread').
         string = 'import %s\n' % element + string
@@ -193,7 +203,10 @@ def _evaluate_for_statement_string(module_context, string):
     # will be impossible to use `...` (Ellipsis) as a token. Docstring types
     # don't need to conform with the current grammar.
     grammar = module_context.evaluator.latest_grammar
-    module = grammar.parse(code.format(indent_block(string)))
+    try:
+        module = grammar.parse(code.format(indent_block(string)), error_recovery=False)
+    except ParserSyntaxError:
+        return []
     try:
         funcdef = next(module.iter_funcdefs())
         # First pick suite, then simple_stmt and then the node,
@@ -202,7 +215,10 @@ def _evaluate_for_statement_string(module_context, string):
     except (AttributeError, IndexError):
         return []
 
-    from jedi.evaluate.representation import FunctionContext
+    if stmt.type not in ('name', 'atom', 'atom_expr'):
+        return []
+
+    from jedi.evaluate.context import FunctionContext
     function_context = FunctionContext(
         module_context.evaluator,
         module_context,
@@ -223,7 +239,10 @@ def _execute_types_in_stmt(module_context, stmt):
     contain is executed. (Used as type information).
     """
     definitions = module_context.eval_node(stmt)
-    return unite(_execute_array_values(module_context.evaluator, d) for d in definitions)
+    return ContextSet.from_sets(
+        _execute_array_values(module_context.evaluator, d)
+        for d in definitions
+    )
 
 
 def _execute_array_values(evaluator, array):
@@ -231,22 +250,27 @@ def _execute_array_values(evaluator, array):
     Tuples indicate that there's not just one return value, but the listed
     ones.  `(str, int)` means that it returns a tuple with both types.
     """
+    from jedi.evaluate.context.iterable import SequenceLiteralContext, FakeSequence
     if isinstance(array, SequenceLiteralContext):
         values = []
         for lazy_context in array.py__iter__():
-            objects = unite(_execute_array_values(evaluator, typ) for typ in lazy_context.infer())
-            values.append(context.LazyKnownContexts(objects))
-        return set([FakeSequence(evaluator, array.array_type, values)])
+            objects = ContextSet.from_sets(
+                _execute_array_values(evaluator, typ)
+                for typ in lazy_context.infer()
+            )
+            values.append(LazyKnownContexts(objects))
+        return {FakeSequence(evaluator, array.array_type, values)}
     else:
         return array.execute_evaluated()
 
 
 @evaluator_method_cache()
 def infer_param(execution_context, param):
-    from jedi.evaluate.instance import AnonymousInstanceFunctionExecution
+    from jedi.evaluate.context.instance import InstanceArguments
+    from jedi.evaluate.context import FunctionExecutionContext
 
     def eval_docstring(docstring):
-        return set(
+        return ContextSet.from_iterable(
             p
             for param_str in _search_param_in_docstr(docstring, param.name.value)
             for p in _evaluate_for_statement_string(module_context, param_str)
@@ -254,18 +278,20 @@ def infer_param(execution_context, param):
     module_context = execution_context.get_root_context()
     func = param.get_parent_function()
     if func.type == 'lambdef':
-        return set()
+        return NO_CONTEXTS
 
     types = eval_docstring(execution_context.py__doc__())
-    if isinstance(execution_context, AnonymousInstanceFunctionExecution) and \
-            execution_context.function_context.name.string_name == '__init__':
-        class_context = execution_context.instance.class_context
+    if isinstance(execution_context, FunctionExecutionContext) \
+            and isinstance(execution_context.var_args, InstanceArguments) \
+            and execution_context.function_context.py__name__() == '__init__':
+        class_context = execution_context.var_args.instance.class_context
         types |= eval_docstring(class_context.py__doc__())
 
     return types
 
 
 @evaluator_method_cache()
+@iterator_to_context_set
 def infer_return_types(function_context):
     def search_return_in_docstr(code):
         for p in DOCSTRING_RETURN_PATTERNS:
@@ -279,4 +305,3 @@ def infer_return_types(function_context):
     for type_str in search_return_in_docstr(function_context.py__doc__()):
         for type_eval in _evaluate_for_statement_string(function_context.get_root_context(), type_str):
             yield type_eval
-

jedi/evaluate/dynamic.py

@@ -14,7 +14,7 @@ It works as follows:
 
 - |Jedi| sees a param
 - search for function calls named ``foo``
-- execute these calls and check the input. This work with a ``ParamListener``.
+- execute these calls and check the input.
 """
 
 from parso.python import tree
@@ -22,35 +22,36 @@ from jedi import settings
 from jedi import debug
 from jedi.evaluate.cache import evaluator_function_cache
 from jedi.evaluate import imports
-from jedi.evaluate.param import TreeArguments, create_default_params
+from jedi.evaluate.arguments import TreeArguments
+from jedi.evaluate.param import create_default_params
 from jedi.evaluate.helpers import is_stdlib_path
-from jedi.common import to_list, unite
+from jedi.evaluate.utils import to_list
 from jedi.parser_utils import get_parent_scope
+from jedi.evaluate.context import ModuleContext, instance
+from jedi.evaluate.base_context import ContextSet, NO_CONTEXTS
+from jedi.evaluate import recursion
 
 
 MAX_PARAM_SEARCHES = 20
 
 
-class ParamListener(object):
-    """
-    This listener is used to get the params for a function.
-    """
-    def __init__(self):
-        self.param_possibilities = []
-
-    def execute(self, params):
-        self.param_possibilities += params
-
-
-class MergedExecutedParams(object):
+class DynamicExecutedParams(object):
     """
     Simulates being a parameter while actually just being multiple params.
     """
-    def __init__(self, executed_params):
+
+    def __init__(self, evaluator, executed_params):
+        self.evaluator = evaluator
         self._executed_params = executed_params
 
     def infer(self):
-        return unite(p.infer() for p in self._executed_params)
+        with recursion.execution_allowed(self.evaluator, self) as allowed:
+            # We need to catch recursions that may occur, because an
+            # anonymous functions can create an anonymous parameter that is
+            # more or less self referencing.
+            if allowed:
+                return ContextSet.from_sets(p.infer() for p in self._executed_params)
+            return NO_CONTEXTS
 
 
 @debug.increase_indent
@@ -80,52 +81,58 @@ def search_params(evaluator, execution_context, funcdef):
             # you will see the slowdown, especially in 3.6.
             return create_default_params(execution_context, funcdef)
 
-        debug.dbg('Dynamic param search in %s.', funcdef.name.value, color='MAGENTA')
-
-        module_context = execution_context.get_root_context()
-        function_executions = _search_function_executions(
-            evaluator,
-            module_context,
-            funcdef
-        )
-        if function_executions:
-            zipped_params = zip(*list(
-                function_execution.get_params()
-                for function_execution in function_executions
-            ))
-            params = [MergedExecutedParams(executed_params) for executed_params in zipped_params]
-            # Evaluate the ExecutedParams to types.
+        if funcdef.type == 'lambdef':
+            string_name = _get_lambda_name(funcdef)
+            if string_name is None:
+                return create_default_params(execution_context, funcdef)
         else:
-            return create_default_params(execution_context, funcdef)
-        debug.dbg('Dynamic param result finished', color='MAGENTA')
+            string_name = funcdef.name.value
+        debug.dbg('Dynamic param search in %s.', string_name, color='MAGENTA')
+
+        try:
+            module_context = execution_context.get_root_context()
+            function_executions = _search_function_executions(
+                evaluator,
+                module_context,
+                funcdef,
+                string_name=string_name,
+            )
+            if function_executions:
+                zipped_params = zip(*list(
+                    function_execution.get_executed_params()
+                    for function_execution in function_executions
+                ))
+                params = [DynamicExecutedParams(evaluator, executed_params) for executed_params in zipped_params]
+                # Evaluate the ExecutedParams to types.
+            else:
+                return create_default_params(execution_context, funcdef)
+        finally:
+            debug.dbg('Dynamic param result finished', color='MAGENTA')
         return params
     finally:
         evaluator.dynamic_params_depth -= 1
 
 
-@evaluator_function_cache(default=[])
+@evaluator_function_cache(default=None)
 @to_list
-def _search_function_executions(evaluator, module_context, funcdef):
+def _search_function_executions(evaluator, module_context, funcdef, string_name):
     """
     Returns a list of param names.
     """
-    from jedi.evaluate import representation as er
-
-    func_string_name = funcdef.name.value
     compare_node = funcdef
-    if func_string_name == '__init__':
+    if string_name == '__init__':
         cls = get_parent_scope(funcdef)
         if isinstance(cls, tree.Class):
-            func_string_name = cls.name.value
+            string_name = cls.name.value
             compare_node = cls
 
     found_executions = False
     i = 0
     for for_mod_context in imports.get_modules_containing_name(
-            evaluator, [module_context], func_string_name):
-        if not isinstance(module_context, er.ModuleContext):
+            evaluator, [module_context], string_name):
+        if not isinstance(module_context, ModuleContext):
             return
-        for name, trailer in _get_possible_nodes(for_mod_context, func_string_name):
+        for name, trailer in _get_possible_nodes(for_mod_context, string_name):
             i += 1
 
             # This is a simple way to stop Jedi's dynamic param recursion
@@ -146,6 +153,18 @@ def _search_function_executions(evaluator, module_context, funcdef):
             return
 
 
+def _get_lambda_name(node):
+    stmt = node.parent
+    if stmt.type == 'expr_stmt':
+        first_operator = next(stmt.yield_operators(), None)
+        if first_operator == '=':
+            first = stmt.children[0]
+            if first.type == 'name':
+                return first.value
+
+    return None
+
+
 def _get_possible_nodes(module_context, func_string_name):
     try:
         names = module_context.tree_node.get_used_names()[func_string_name]
@@ -160,16 +179,14 @@ def _get_possible_nodes(module_context, func_string_name):
 
 
 def _check_name_for_execution(evaluator, context, compare_node, name, trailer):
-    from jedi.evaluate import representation as er, instance
+    from jedi.evaluate.context.function import FunctionExecutionContext
 
     def create_func_excs():
         arglist = trailer.children[1]
         if arglist == ')':
-            arglist = ()
+            arglist = None
         args = TreeArguments(evaluator, context, arglist, trailer)
-        if value_node.type == 'funcdef':
-            yield value.get_function_execution(args)
-        else:
+        if value_node.type == 'classdef':
             created_instance = instance.TreeInstance(
                 evaluator,
                 value.parent_context,
@@ -178,18 +195,20 @@ def _check_name_for_execution(evaluator, context, compare_node, name, trailer):
             )
             for execution in created_instance.create_init_executions():
                 yield execution
+        else:
+            yield value.get_function_execution(args)
 
     for value in evaluator.goto_definitions(context, name):
         value_node = value.tree_node
         if compare_node == value_node:
             for func_execution in create_func_excs():
                 yield func_execution
-        elif isinstance(value.parent_context, er.FunctionExecutionContext) and \
+        elif isinstance(value.parent_context, FunctionExecutionContext) and \
                 compare_node.type == 'funcdef':
             # Here we're trying to find decorators by checking the first
             # parameter. It's not very generic though. Should find a better
             # solution that also applies to nested decorators.
-            params = value.parent_context.get_params()
+            params = value.parent_context.get_executed_params()
             if len(params) != 1:
                 continue
             values = params[0].infer()

jedi/evaluate/filters.py

@@ -5,9 +5,13 @@ are needed for name resolution.
 from abc import abstractmethod
 
 from parso.tree import search_ancestor
+
+from jedi._compatibility import use_metaclass, Parameter
+from jedi.cache import memoize_method
 from jedi.evaluate import flow_analysis
-from jedi.common import to_list, unite
+from jedi.evaluate.base_context import ContextSet, Context
 from jedi.parser_utils import get_parent_scope
+from jedi.evaluate.utils import to_list
 
 
 class AbstractNameDefinition(object):
@@ -24,7 +28,7 @@ class AbstractNameDefinition(object):
     def goto(self):
         # Typically names are already definitions and therefore a goto on that
         # name will always result on itself.
-        return set([self])
+        return {self}
 
     def get_root_context(self):
         return self.parent_context.get_root_context()
@@ -34,11 +38,8 @@ class AbstractNameDefinition(object):
             return '<%s: %s>' % (self.__class__.__name__, self.string_name)
         return '<%s: %s@%s>' % (self.__class__.__name__, self.string_name, self.start_pos)
 
-    def execute(self, arguments):
-        return unite(context.execute(arguments) for context in self.infer())
-
-    def execute_evaluated(self, *args, **kwargs):
-        return unite(context.execute_evaluated(*args, **kwargs) for context in self.infer())
+    def is_import(self):
+        return False
 
     @property
     def api_type(self):
@@ -53,6 +54,10 @@ class AbstractTreeName(AbstractNameDefinition):
     def goto(self):
         return self.parent_context.evaluator.goto(self.parent_context, self.tree_name)
 
+    def is_import(self):
+        imp = search_ancestor(self.tree_name, 'import_from', 'import_name')
+        return imp is not None
+
     @property
     def string_name(self):
         return self.tree_name.value
@@ -64,7 +69,7 @@ class AbstractTreeName(AbstractNameDefinition):
 
 class ContextNameMixin(object):
     def infer(self):
-        return set([self._context])
+        return ContextSet(self._context)
 
     def get_root_context(self):
         if self.parent_context is None:
@@ -93,8 +98,8 @@ class TreeNameDefinition(AbstractTreeName):
 
     def infer(self):
         # Refactor this, should probably be here.
-        from jedi.evaluate.finder import _name_to_types
-        return _name_to_types(self.parent_context.evaluator, self.parent_context, self.tree_name)
+        from jedi.evaluate.syntax_tree import tree_name_to_contexts
+        return tree_name_to_contexts(self.parent_context.evaluator, self.parent_context, self.tree_name)
 
     @property
     def api_type(self):
@@ -105,34 +110,37 @@ class TreeNameDefinition(AbstractTreeName):
 
 
 class ParamName(AbstractTreeName):
-    api_type = 'param'
+    api_type = u'param'
 
     def __init__(self, parent_context, tree_name):
         self.parent_context = parent_context
         self.tree_name = tree_name
 
+    def get_kind(self):
+        tree_param = search_ancestor(self.tree_name, 'param')
+        if tree_param.star_count == 1:  # *args
+            return Parameter.VAR_POSITIONAL
+        if tree_param.star_count == 2:  # **kwargs
+            return Parameter.VAR_KEYWORD
+
+        parent = tree_param.parent
+        for p in parent.children:
+            if p.type == 'param':
+                if p.star_count:
+                    return Parameter.KEYWORD_ONLY
+                if p == tree_param:
+                    break
+        return Parameter.POSITIONAL_OR_KEYWORD
+
     def infer(self):
         return self.get_param().infer()
 
     def get_param(self):
-        params = self.parent_context.get_params()
+        params = self.parent_context.get_executed_params()
         param_node = search_ancestor(self.tree_name, 'param')
         return params[param_node.position_index]
 
 
-class AnonymousInstanceParamName(ParamName):
-    def infer(self):
-        param_node = search_ancestor(self.tree_name, 'param')
-        # TODO I think this should not belong here. It's not even really true,
-        #      because classmethod and other descriptors can change it.
-        if param_node.position_index == 0:
-            # This is a speed optimization, to return the self param (because
-            # it's known). This only affects anonymous instances.
-            return set([self.parent_context.instance])
-        else:
-            return self.get_param().infer()
-
-
 class AbstractFilter(object):
     _until_position = None
 
@@ -160,7 +168,7 @@ class AbstractUsedNamesFilter(AbstractFilter):
 
     def get(self, name):
         try:
-            names = self._used_names[str(name)]
+            names = self._used_names[name]
         except KeyError:
             return []
 
@@ -210,7 +218,10 @@ class ParserTreeFilter(AbstractUsedNamesFilter):
     def _check_flows(self, names):
         for name in sorted(names, key=lambda name: name.start_pos, reverse=True):
             check = flow_analysis.reachability_check(
-                self._node_context, self._parser_scope, name, self._origin_scope
+                context=self._node_context,
+                context_scope=self._parser_scope,
+                node=name,
+                origin_scope=self._origin_scope
             )
             if check is not flow_analysis.UNREACHABLE:
                 yield name
@@ -242,10 +253,6 @@ class FunctionExecutionFilter(ParserTreeFilter):
                 yield TreeNameDefinition(self.context, name)
 
 
-class AnonymousInstanceFunctionExecutionFilter(FunctionExecutionFilter):
-    param_name = AnonymousInstanceParamName
-
-
 class GlobalNameFilter(AbstractUsedNamesFilter):
     def __init__(self, context, parser_scope):
         super(GlobalNameFilter, self).__init__(context, parser_scope)
@@ -263,19 +270,162 @@ class DictFilter(AbstractFilter):
 
     def get(self, name):
         try:
-            value = self._convert(name, self._dct[str(name)])
+            value = self._convert(name, self._dct[name])
         except KeyError:
             return []
-
-        return list(self._filter([value]))
+        else:
+            return list(self._filter([value]))
 
     def values(self):
-        return self._filter(self._convert(*item) for item in self._dct.items())
+        def yielder():
+            for item in self._dct.items():
+                try:
+                    yield self._convert(*item)
+                except KeyError:
+                    pass
+        return self._filter(yielder())
 
     def _convert(self, name, value):
         return value
 
 
+class MergedFilter(object):
+    def __init__(self, *filters):
+        self._filters = filters
+
+    def get(self, name):
+        return [n for filter in self._filters for n in filter.get(name)]
+
+    def values(self):
+        return [n for filter in self._filters for n in filter.values()]
+
+    def __repr__(self):
+        return '%s(%s)' % (self.__class__.__name__, ', '.join(str(f) for f in self._filters))
+
+
+class _BuiltinMappedMethod(Context):
+    """``Generator.__next__`` ``dict.values`` methods and so on."""
+    api_type = u'function'
+
+    def __init__(self, builtin_context, method, builtin_func):
+        super(_BuiltinMappedMethod, self).__init__(
+            builtin_context.evaluator,
+            parent_context=builtin_context
+        )
+        self._method = method
+        self._builtin_func = builtin_func
+
+    def py__call__(self, params):
+        # TODO add TypeError if params are given/or not correct.
+        return self._method(self.parent_context)
+
+    def __getattr__(self, name):
+        return getattr(self._builtin_func, name)
+
+
+class SpecialMethodFilter(DictFilter):
+    """
+    A filter for methods that are defined in this module on the corresponding
+    classes like Generator (for __next__, etc).
+    """
+    class SpecialMethodName(AbstractNameDefinition):
+        api_type = u'function'
+
+        def __init__(self, parent_context, string_name, value, builtin_context):
+            callable_, python_version = value
+            if python_version is not None and \
+                    python_version != parent_context.evaluator.environment.version_info.major:
+                raise KeyError
+
+            self.parent_context = parent_context
+            self.string_name = string_name
+            self._callable = callable_
+            self._builtin_context = builtin_context
+
+        def infer(self):
+            for filter in self._builtin_context.get_filters():
+                # We can take the first index, because on builtin methods there's
+                # always only going to be one name. The same is true for the
+                # inferred values.
+                for name in filter.get(self.string_name):
+                    builtin_func = next(iter(name.infer()))
+                    break
+                else:
+                    continue
+                break
+            return ContextSet(
+                _BuiltinMappedMethod(self.parent_context, self._callable, builtin_func)
+            )
+
+    def __init__(self, context, dct, builtin_context):
+        super(SpecialMethodFilter, self).__init__(dct)
+        self.context = context
+        self._builtin_context = builtin_context
+        """
+        This context is what will be used to introspect the name, where as the
+        other context will be used to execute the function.
+
+        We distinguish, because we have to.
+        """
+
+    def _convert(self, name, value):
+        return self.SpecialMethodName(self.context, name, value, self._builtin_context)
+
+
+class _OverwriteMeta(type):
+    def __init__(cls, name, bases, dct):
+        super(_OverwriteMeta, cls).__init__(name, bases, dct)
+
+        base_dct = {}
+        for base_cls in reversed(cls.__bases__):
+            try:
+                base_dct.update(base_cls.overwritten_methods)
+            except AttributeError:
+                pass
+
+        for func in cls.__dict__.values():
+            try:
+                base_dct.update(func.registered_overwritten_methods)
+            except AttributeError:
+                pass
+        cls.overwritten_methods = base_dct
+
+
+class AbstractObjectOverwrite(use_metaclass(_OverwriteMeta, object)):
+    def get_object(self):
+        raise NotImplementedError
+
+    def get_filters(self, search_global, *args, **kwargs):
+        yield SpecialMethodFilter(self, self.overwritten_methods, self.get_object())
+
+        for filter in self.get_object().get_filters(search_global):
+            yield filter
+
+
+class BuiltinOverwrite(Context, AbstractObjectOverwrite):
+    special_object_identifier = None
+
+    def __init__(self, evaluator):
+        super(BuiltinOverwrite, self).__init__(evaluator, evaluator.builtins_module)
+
+    @memoize_method
+    def get_object(self):
+        from jedi.evaluate import compiled
+        assert self.special_object_identifier
+        return compiled.get_special_object(self.evaluator, self.special_object_identifier)
+
+    def py__class__(self):
+        return self.get_object().py__class__()
+
+
+def publish_method(method_name, python_version_match=None):
+    def decorator(func):
+        dct = func.__dict__.setdefault('registered_overwritten_methods', {})
+        dct[method_name] = func, python_version_match
+        return func
+    return decorator
+
+
 def get_global_filters(evaluator, context, until_position, origin_scope):
     """
     Returns all filters in order of priority for name resolution.
@@ -290,43 +440,40 @@ def get_global_filters(evaluator, context, until_position, origin_scope):
     ... def func():
     ...     y = None
     ... '''))
-    >>> module_node = script._get_module_node()
+    >>> module_node = script._module_node
     >>> scope = next(module_node.iter_funcdefs())
     >>> scope
     <Function: func@3-5>
     >>> context = script._get_module().create_context(scope)
     >>> filters = list(get_global_filters(context.evaluator, context, (4, 0), None))
 
-    First we get the names names from the function scope.
+    First we get the names from the function scope.
 
-    >>> no_unicode_pprint(filters[0])
-    <ParserTreeFilter: <ModuleContext: @2-5>>
+    >>> no_unicode_pprint(filters[0])                    #doctest: +ELLIPSIS
+    MergedFilter(<ParserTreeFilter: ...>, <GlobalNameFilter: ...>)
     >>> sorted(str(n) for n in filters[0].values())
     ['<TreeNameDefinition: func@(3, 4)>', '<TreeNameDefinition: x@(2, 0)>']
-    >>> filters[0]._until_position
+    >>> filters[0]._filters[0]._until_position
     (4, 0)
+    >>> filters[0]._filters[1]._until_position
 
     Then it yields the names from one level "lower". In this example, this is
-    the module scope. As a side note, you can see, that the position in the
-    filter is now None, because typically the whole module is loaded before the
-    function is called.
+    the module scope (including globals).
+    As a side note, you can see, that the position in the filter is None on the
+    globals filter, because there the whole module is searched.
 
-    >>> filters[1].values()  # global names -> there are none in our example.
+    >>> list(filters[1].values())  # package modules -> Also empty.
     []
-    >>> list(filters[2].values())  # package modules -> Also empty.
-    []
-    >>> sorted(name.string_name for name in filters[3].values())  # Module attributes
+    >>> sorted(name.string_name for name in filters[2].values())  # Module attributes
     ['__doc__', '__file__', '__name__', '__package__']
-    >>> print(filters[1]._until_position)
-    None
 
     Finally, it yields the builtin filter, if `include_builtin` is
     true (default).
 
-    >>> filters[4].values()                              #doctest: +ELLIPSIS
+    >>> filters[3].values()                              #doctest: +ELLIPSIS
     [<CompiledName: ...>, ...]
     """
-    from jedi.evaluate.representation import FunctionExecutionContext
+    from jedi.evaluate.context.function import FunctionExecutionContext
     while context is not None:
         # Names in methods cannot be resolved within the class.
         for filter in context.get_filters(
@@ -341,5 +488,5 @@ def get_global_filters(evaluator, context, until_position, origin_scope):
         context = context.parent_context
 
     # Add builtins to the global scope.
-    for filter in evaluator.BUILTINS.get_filters(search_global=True):
+    for filter in evaluator.builtins_module.get_filters(search_global=True):
         yield filter

jedi/evaluate/finder.py

@@ -18,20 +18,16 @@ check for -> a is a string). There's big potential in these checks.
 from parso.python import tree
 from parso.tree import search_ancestor
 from jedi import debug
-from jedi.common import unite
 from jedi import settings
-from jedi.evaluate import representation as er
-from jedi.evaluate.instance import AbstractInstanceContext
+from jedi.evaluate.context import AbstractInstanceContext
 from jedi.evaluate import compiled
-from jedi.evaluate import pep0484
-from jedi.evaluate import iterable
-from jedi.evaluate import imports
 from jedi.evaluate import analysis
 from jedi.evaluate import flow_analysis
-from jedi.evaluate import param
+from jedi.evaluate.arguments import TreeArguments
 from jedi.evaluate import helpers
+from jedi.evaluate.context import iterable
 from jedi.evaluate.filters import get_global_filters, TreeNameDefinition
-from jedi.evaluate.context import ContextualizedName, ContextualizedNode
+from jedi.evaluate.base_context import ContextSet
 from jedi.parser_utils import is_scope, get_parent_scope
 
 
@@ -60,9 +56,12 @@ class NameFinder(object):
         names = self.filter_name(filters)
         if self._found_predefined_types is not None and names:
             check = flow_analysis.reachability_check(
-                self._context, self._context.tree_node, self._name)
+                context=self._context,
+                context_scope=self._context.tree_node,
+                node=self._name,
+            )
             if check is flow_analysis.UNREACHABLE:
-                return set()
+                return ContextSet()
             return self._found_predefined_types
 
         types = self._names_to_types(names, attribute_lookup)
@@ -96,7 +95,26 @@ class NameFinder(object):
     def get_filters(self, search_global=False):
         origin_scope = self._get_origin_scope()
         if search_global:
-            return get_global_filters(self._evaluator, self._context, self._position, origin_scope)
+            position = self._position
+
+            # For functions and classes the defaults don't belong to the
+            # function and get evaluated in the context before the function. So
+            # make sure to exclude the function/class name.
+            if origin_scope is not None:
+                ancestor = search_ancestor(origin_scope, 'funcdef', 'classdef', 'lambdef')
+                lambdef = None
+                if ancestor == 'lambdef':
+                    # For lambdas it's even more complicated since parts will
+                    # be evaluated later.
+                    lambdef = ancestor
+                    ancestor = search_ancestor(origin_scope, 'funcdef', 'classdef')
+                if ancestor is not None:
+                    colon = ancestor.children[-2]
+                    if position < colon.start_pos:
+                        if lambdef is None or position < lambdef.children[-2].start_pos:
+                            position = ancestor.start_pos
+
+            return get_global_filters(self._evaluator, self._context, position, origin_scope)
         else:
             return self._context.get_filters(search_global, self._position, origin_scope=origin_scope)
 
@@ -106,8 +124,7 @@ class NameFinder(object):
         ``filters``), until a name fits.
         """
         names = []
-        if self._context.predefined_names:
-            # TODO is this ok? node might not always be a tree.Name
+        if self._context.predefined_names and isinstance(self._name, tree.Name):
             node = self._name
             while node is not None and not is_scope(node):
                 node = node.parent
@@ -137,14 +154,14 @@ class NameFinder(object):
                                 continue
                 break
 
-        debug.dbg('finder.filter_name "%s" in (%s): %s@%s', self._string_name,
-                  self._context, names, self._position)
+        debug.dbg('finder.filter_name %s in (%s): %s@%s',
+                  self._string_name, self._context, names, self._position)
         return list(names)
 
     def _check_getattr(self, inst):
         """Checks for both __getattr__ and __getattribute__ methods"""
         # str is important, because it shouldn't be `Name`!
-        name = compiled.create(self._evaluator, self._string_name)
+        name = compiled.create_simple_object(self._evaluator, self._string_name)
 
         # This is a little bit special. `__getattribute__` is in Python
         # executed before `__getattr__`. But: I know no use case, where
@@ -153,27 +170,25 @@ class NameFinder(object):
         # We are inversing this, because a hand-crafted `__getattribute__`
         # could still call another hand-crafted `__getattr__`, but not the
         # other way around.
-        names = (inst.get_function_slot_names('__getattr__') or
-                 inst.get_function_slot_names('__getattribute__'))
+        names = (inst.get_function_slot_names(u'__getattr__') or
+                 inst.get_function_slot_names(u'__getattribute__'))
         return inst.execute_function_slots(names, name)
 
     def _names_to_types(self, names, attribute_lookup):
-        types = set()
+        contexts = ContextSet.from_sets(name.infer() for name in names)
 
-        types = unite(name.infer() for name in names)
-
-        debug.dbg('finder._names_to_types: %s -> %s', names, types)
+        debug.dbg('finder._names_to_types: %s -> %s', names, contexts)
         if not names and isinstance(self._context, AbstractInstanceContext):
             # handling __getattr__ / __getattribute__
             return self._check_getattr(self._context)
 
         # Add isinstance and other if/assert knowledge.
-        if not types and isinstance(self._name, tree.Name) and \
+        if not contexts and isinstance(self._name, tree.Name) and \
                 not isinstance(self._name_context, AbstractInstanceContext):
             flow_scope = self._name
             base_node = self._name_context.tree_node
             if base_node.type == 'comp_for':
-                return types
+                return contexts
             while True:
                 flow_scope = get_parent_scope(flow_scope, include_flows=True)
                 n = _check_flow_information(self._name_context, flow_scope,
@@ -182,132 +197,7 @@ class NameFinder(object):
                     return n
                 if flow_scope == base_node:
                     break
-        return types
-
-
-def _name_to_types(evaluator, context, tree_name):
-    types = []
-    node = tree_name.get_definition(import_name_always=True)
-    if node is None:
-        node = tree_name.parent
-        if node.type == 'global_stmt':
-            context = evaluator.create_context(context, tree_name)
-            finder = NameFinder(evaluator, context, context, tree_name.value)
-            filters = finder.get_filters(search_global=True)
-            # For global_stmt lookups, we only need the first possible scope,
-            # which means the function itself.
-            filters = [next(filters)]
-            return finder.find(filters, attribute_lookup=False)
-        elif node.type not in ('import_from', 'import_name'):
-            raise ValueError("Should not happen.")
-
-    typ = node.type
-    if typ == 'for_stmt':
-        types = pep0484.find_type_from_comment_hint_for(context, node, tree_name)
-        if types:
-            return types
-    if typ == 'with_stmt':
-        types = pep0484.find_type_from_comment_hint_with(context, node, tree_name)
-        if types:
-            return types
-    if typ in ('for_stmt', 'comp_for'):
-        try:
-            types = context.predefined_names[node][tree_name.value]
-        except KeyError:
-            cn = ContextualizedNode(context, node.children[3])
-            for_types = iterable.py__iter__types(evaluator, cn.infer(), cn)
-            c_node = ContextualizedName(context, tree_name)
-            types = check_tuple_assignments(evaluator, c_node, for_types)
-    elif typ == 'expr_stmt':
-        types = _remove_statements(evaluator, context, node, tree_name)
-    elif typ == 'with_stmt':
-        context_managers = context.eval_node(node.get_test_node_from_name(tree_name))
-        enter_methods = unite(
-            context_manager.py__getattribute__('__enter__')
-            for context_manager in context_managers
-        )
-        types = unite(method.execute_evaluated() for method in enter_methods)
-    elif typ in ('import_from', 'import_name'):
-        types = imports.infer_import(context, tree_name)
-    elif typ in ('funcdef', 'classdef'):
-        types = _apply_decorators(evaluator, context, node)
-    elif typ == 'try_stmt':
-        # TODO an exception can also be a tuple. Check for those.
-        # TODO check for types that are not classes and add it to
-        # the static analysis report.
-        exceptions = context.eval_node(tree_name.get_previous_sibling().get_previous_sibling())
-        types = unite(
-            evaluator.execute(t, param.ValuesArguments([]))
-            for t in exceptions
-        )
-    else:
-        raise ValueError("Should not happen.")
-    return types
-
-
-def _apply_decorators(evaluator, context, node):
-    """
-    Returns the function, that should to be executed in the end.
-    This is also the places where the decorators are processed.
-    """
-    if node.type == 'classdef':
-        decoratee_context = er.ClassContext(
-            evaluator,
-            parent_context=context,
-            classdef=node
-        )
-    else:
-        decoratee_context = er.FunctionContext(
-            evaluator,
-            parent_context=context,
-            funcdef=node
-        )
-    initial = values = set([decoratee_context])
-    for dec in reversed(node.get_decorators()):
-        debug.dbg('decorator: %s %s', dec, values)
-        dec_values = context.eval_node(dec.children[1])
-        trailer_nodes = dec.children[2:-1]
-        if trailer_nodes:
-            # Create a trailer and evaluate it.
-            trailer = tree.PythonNode('trailer', trailer_nodes)
-            trailer.parent = dec
-            dec_values = evaluator.eval_trailer(context, dec_values, trailer)
-
-        if not len(dec_values):
-            debug.warning('decorator not found: %s on %s', dec, node)
-            return initial
-
-        values = unite(dec_value.execute(param.ValuesArguments([values]))
-                       for dec_value in dec_values)
-        if not len(values):
-            debug.warning('not possible to resolve wrappers found %s', node)
-            return initial
-
-        debug.dbg('decorator end %s', values)
-    return values
-
-
-def _remove_statements(evaluator, context, stmt, name):
-    """
-    This is the part where statements are being stripped.
-
-    Due to lazy evaluation, statements like a = func; b = a; b() have to be
-    evaluated.
-    """
-    types = set()
-    check_instance = None
-
-    pep0484types = \
-        pep0484.find_type_from_comment_hint_assign(context, stmt, name)
-    if pep0484types:
-        return pep0484types
-    types |= context.eval_stmt(stmt, seek_name=name)
-
-    if check_instance is not None:
-        # class renames
-        types = set([er.get_instance_el(evaluator, check_instance, a, True)
-                     if isinstance(a, er.Function) else a for a in types])
-    return types
+        return contexts
 
 
 def _check_flow_information(context, flow, search_name, pos):
@@ -362,7 +252,7 @@ def _check_isinstance_type(context, element, search_name):
 
         # arglist stuff
         arglist = trailer.children[1]
-        args = param.TreeArguments(context.evaluator, context, arglist, trailer)
+        args = TreeArguments(context.evaluator, context, arglist, trailer)
         param_list = list(args.unpack())
         # Disallow keyword arguments
         assert len(param_list) == 2
@@ -377,34 +267,12 @@ def _check_isinstance_type(context, element, search_name):
     except AssertionError:
         return None
 
-    result = set()
+    context_set = ContextSet()
     for cls_or_tup in lazy_context_cls.infer():
-        if isinstance(cls_or_tup, iterable.AbstractSequence) and \
-                cls_or_tup.array_type == 'tuple':
+        if isinstance(cls_or_tup, iterable.Sequence) and cls_or_tup.array_type == 'tuple':
             for lazy_context in cls_or_tup.py__iter__():
                 for context in lazy_context.infer():
-                    result |= context.execute_evaluated()
+                    context_set |= context.execute_evaluated()
         else:
-            result |= cls_or_tup.execute_evaluated()
-    return result
-
-
-def check_tuple_assignments(evaluator, contextualized_name, types):
-    """
-    Checks if tuples are assigned.
-    """
-    lazy_context = None
-    for index, node in contextualized_name.assignment_indexes():
-        cn = ContextualizedNode(contextualized_name.context, node)
-        iterated = iterable.py__iter__(evaluator, types, cn)
-        for _ in range(index + 1):
-            try:
-                lazy_context = next(iterated)
-            except StopIteration:
-                # We could do this with the default param in next. But this
-                # would allow this loop to run for a very long time if the
-                # index number is high. Therefore break if the loop is
-                # finished.
-                return set()
-        types = lazy_context.infer()
-    return types
+            context_set |= cls_or_tup.execute_evaluated()
+    return context_set

jedi/evaluate/flow_analysis.py

@@ -1,4 +1,5 @@
 from jedi.parser_utils import get_flow_branch_keyword, is_scope, get_parent_scope
+from jedi.evaluate.recursion import execution_allowed
 
 
 class Status(object):
@@ -59,7 +60,8 @@ def reachability_check(context, context_scope, node, origin_scope=None):
                     if not branch_matches and origin_keyword == 'else' \
                             and node_keyword == 'except':
                         return UNREACHABLE
-                break
+                if branch_matches:
+                    break
 
         # Direct parents get resolved, we filter scopes that are separate
         # branches.  This makes sense for autocompletion and static analysis.
@@ -104,9 +106,13 @@ def _break_check(context, context_scope, flow_scope, node):
 
 
 def _check_if(context, node):
-    types = context.eval_node(node)
-    values = set(x.py__bool__() for x in types)
-    if len(values) == 1:
-        return Status.lookup_table[values.pop()]
-    else:
-        return UNSURE
+    with execution_allowed(context.evaluator, node) as allowed:
+        if not allowed:
+            return UNSURE
+
+        types = context.eval_node(node)
+        values = set(x.py__bool__() for x in types)
+        if len(values) == 1:
+            return Status.lookup_table[values.pop()]
+        else:
+            return UNSURE

jedi/evaluate/helpers.py

@@ -6,6 +6,8 @@ from itertools import chain
 from contextlib import contextmanager
 
 from parso.python import tree
+
+from jedi._compatibility import unicode
 from jedi.parser_utils import get_parent_scope
 
 
@@ -17,7 +19,7 @@ def is_stdlib_path(path):
         return False
 
     base_path = os.path.join(sys.prefix, 'lib', 'python')
-    return bool(re.match(re.escape(base_path) + '\d.\d', path))
+    return bool(re.match(re.escape(base_path) + r'\d.\d', path))
 
 
 def deep_ast_copy(obj):
@@ -55,10 +57,17 @@ def evaluate_call_of_leaf(context, leaf, cut_own_trailer=False):
 
     If you're using the leaf, e.g. the bracket `)` it will return ``list([])``.
 
-    # TODO remove cut_own_trailer option, since its always used with it. Just
-    #      ignore it, It's not what we want anyway. Or document it better?
+    We use this function for two purposes. Given an expression ``bar.foo``,
+    we may want to
+      - infer the type of ``foo`` to offer completions after foo
+      - infer the type of ``bar`` to be able to jump to the definition of foo
+    The option ``cut_own_trailer`` must be set to true for the second purpose.
     """
     trailer = leaf.parent
+    if trailer.type == 'fstring':
+        from jedi.evaluate import compiled
+        return compiled.get_string_context_set(context.evaluator)
+
     # The leaf may not be the last or first child, because there exist three
     # different trailers: `( x )`, `[ x ]` and `.x`. In the first two examples
     # we should not match anything more than x.
@@ -86,9 +95,14 @@ def evaluate_call_of_leaf(context, leaf, cut_own_trailer=False):
         base = power.children[0]
         trailers = power.children[1:cut]
 
+    if base == 'await':
+        base = trailers[0]
+        trailers = trailers[1:]
+
     values = context.eval_node(base)
+    from jedi.evaluate.syntax_tree import eval_trailer
     for trailer in trailers:
-        values = context.eval_trailer(values, trailer)
+        values = eval_trailer(context, values, trailer)
     return values
 
 
@@ -165,10 +179,61 @@ def get_module_names(module, all_scopes):
 @contextmanager
 def predefine_names(context, flow_scope, dct):
     predefined = context.predefined_names
-    if flow_scope in predefined:
-        raise NotImplementedError('Why does this happen?')
     predefined[flow_scope] = dct
     try:
         yield
     finally:
         del predefined[flow_scope]
+
+
+def is_compiled(context):
+    from jedi.evaluate.compiled import CompiledObject
+    return isinstance(context, CompiledObject)
+
+
+def is_string(context):
+    if context.evaluator.environment.version_info.major == 2:
+        str_classes = (unicode, bytes)
+    else:
+        str_classes = (unicode,)
+    return is_compiled(context) and isinstance(context.get_safe_value(default=None), str_classes)
+
+
+def is_literal(context):
+    return is_number(context) or is_string(context)
+
+
+def _get_safe_value_or_none(context, accept):
+    if is_compiled(context):
+        value = context.get_safe_value(default=None)
+        if isinstance(value, accept):
+            return value
+
+
+def get_int_or_none(context):
+    return _get_safe_value_or_none(context, int)
+
+
+def is_number(context):
+    return _get_safe_value_or_none(context, (int, float)) is not None
+
+
+class EvaluatorTypeError(Exception):
+    pass
+
+
+class EvaluatorIndexError(Exception):
+    pass
+
+
+class EvaluatorKeyError(Exception):
+    pass
+
+
+@contextmanager
+def reraise_as_evaluator(*exception_classes):
+    try:
+        yield
+    except exception_classes as e:
+        new_exc_cls = globals()['Evaluator' + e.__class__.__name__]
+        raise new_exc_cls(e)

jedi/evaluate/imports.py

@@ -9,33 +9,52 @@ This module uses imp for python up to 3.2 and importlib for python 3.3 on; the
 correct implementation is delegated to _compatibility.
 
 This module also supports import autocompletion, which means to complete
-statements like ``from datetim`` (curser at the end would return ``datetime``).
+statements like ``from datetim`` (cursor at the end would return ``datetime``).
 """
-import imp
 import os
-import pkgutil
-import sys
 
 from parso.python import tree
 from parso.tree import search_ancestor
-from parso.cache import parser_cache
 from parso import python_bytes_to_unicode
 
-from jedi._compatibility import find_module, unicode, ImplicitNSInfo
+from jedi._compatibility import (FileNotFoundError, ImplicitNSInfo,
+                                 force_unicode, unicode)
 from jedi import debug
 from jedi import settings
-from jedi.common import unite
+from jedi.parser_utils import get_cached_code_lines
 from jedi.evaluate import sys_path
 from jedi.evaluate import helpers
 from jedi.evaluate import compiled
 from jedi.evaluate import analysis
+from jedi.evaluate.utils import unite
 from jedi.evaluate.cache import evaluator_method_cache
 from jedi.evaluate.filters import AbstractNameDefinition
+from jedi.evaluate.base_context import ContextSet, NO_CONTEXTS
+
+
+class ModuleCache(object):
+    def __init__(self):
+        self._path_cache = {}
+        self._name_cache = {}
+
+    def add(self, module, name):
+        path = module.py__file__()
+        self._path_cache[path] = module
+        self._name_cache[name] = module
+
+    def iterate_modules_with_names(self):
+        return self._name_cache.items()
+
+    def get(self, name):
+        return self._name_cache[name]
+
+    def get_from_path(self, path):
+        return self._path_cache[path]
 
 
 # This memoization is needed, because otherwise we will infinitely loop on
 # certain imports.
-@evaluator_method_cache(default=set())
+@evaluator_method_cache(default=NO_CONTEXTS)
 def infer_import(context, tree_name, is_goto=False):
     module_context = context.get_root_context()
     import_node = search_ancestor(tree_name, 'import_name', 'import_from')
@@ -63,7 +82,7 @@ def infer_import(context, tree_name, is_goto=False):
     #    scopes = [NestedImportModule(module, import_node)]
 
     if not types:
-        return set()
+        return NO_CONTEXTS
 
     if from_import_name is not None:
         types = unite(
@@ -72,8 +91,11 @@ def infer_import(context, tree_name, is_goto=False):
                 name_context=context,
                 is_goto=is_goto,
                 analysis_errors=False
-            ) for t in types
+            )
+            for t in types
         )
+        if not is_goto:
+            types = ContextSet.from_set(types)
 
         if not types:
             path = import_path + [from_import_name]
@@ -126,20 +148,13 @@ class NestedImportModule(tree.Module):
 
 def _add_error(context, name, message=None):
     # Should be a name, not a string!
+    if message is None:
+        name_str = str(name.value) if isinstance(name, tree.Name) else name
+        message = 'No module named ' + name_str
     if hasattr(name, 'parent'):
         analysis.add(context, 'import-error', name, message)
-
-
-def get_init_path(directory_path):
-    """
-    The __init__ file can be searched in a directory. If found return it, else
-    None.
-    """
-    for suffix, _, _ in imp.get_suffixes():
-        path = os.path.join(directory_path, '__init__' + suffix)
-        if os.path.exists(path):
-            return path
-    return None
+    else:
+        debug.warning('ImportError without origin: ' + message)
 
 
 class ImportName(AbstractNameDefinition):
@@ -200,7 +215,7 @@ class Importer(object):
 
         if level:
             base = module_context.py__package__().split('.')
-            if base == ['']:
+            if base == [''] or base == ['__main__']:
                 base = []
             if level > len(base):
                 path = module_context.py__file__()
@@ -222,10 +237,11 @@ class Importer(object):
                         else:
                             import_path.insert(0, dir_name)
                     else:
-                        _add_error(module_context, import_path[-1])
+                        _add_error(
+                            module_context, import_path[-1],
+                            message='Attempted relative import beyond top-level package.'
+                        )
                         import_path = []
-                        # TODO add import error.
-                        debug.warning('Attempted relative import beyond top-level package.')
                 # If no path is defined in the module we have no ideas where we
                 # are in the file system. Therefore we cannot know what to do.
                 # In this case we just let the path there and ignore that it's
@@ -244,33 +260,27 @@ class Importer(object):
         """Returns the import path as pure strings instead of `Name`."""
         return tuple(
             name.value if isinstance(name, tree.Name) else name
-            for name in self.import_path)
+            for name in self.import_path
+        )
 
     def sys_path_with_modifications(self):
-        in_path = []
-        sys_path_mod = list(sys_path.sys_path_with_modifications(
-            self._evaluator,
-            self.module_context
-        ))
-        if self.file_path is not None:
-            # If you edit e.g. gunicorn, there will be imports like this:
-            # `from gunicorn import something`. But gunicorn is not in the
-            # sys.path. Therefore look if gunicorn is a parent directory, #56.
-            if self.import_path:  # TODO is this check really needed?
-                for path in sys_path.traverse_parents(self.file_path):
-                    if os.path.basename(path) == self.str_import_path[0]:
-                        in_path.append(os.path.dirname(path))
 
-            # Since we know nothing about the call location of the sys.path,
-            # it's a possibility that the current directory is the origin of
-            # the Python execution.
-            sys_path_mod.insert(0, os.path.dirname(self.file_path))
+        sys_path_mod = (
+            self._evaluator.get_sys_path()
+            + sys_path.check_sys_path_modifications(self.module_context)
+        )
 
-        return in_path + sys_path_mod
+        if self.import_path and self.file_path is not None \
+                and self._evaluator.environment.version_info.major == 2:
+            # Python2 uses an old strange way of importing relative imports.
+            sys_path_mod.append(force_unicode(os.path.dirname(self.file_path)))
+
+        return sys_path_mod
 
     def follow(self):
-        if not self.import_path:
-            return set()
+        if not self.import_path or not self._evaluator.infer_enabled:
+            return NO_CONTEXTS
+
         return self._do_import(self.import_path, self.sys_path_with_modifications())
 
     def _do_import(self, import_path, sys_path):
@@ -278,7 +288,7 @@ class Importer(object):
         This method is very similar to importlib's `_gcd_import`.
         """
         import_parts = [
-            i.value if isinstance(i, tree.Name) else i
+            force_unicode(i.value if isinstance(i, tree.Name) else i)
             for i in import_path
         ]
 
@@ -294,9 +304,17 @@ class Importer(object):
                 # Old style
                 return self._do_import(('flaskext',) + import_path[2:], sys_path)
 
+        if import_parts[0] in settings.auto_import_modules:
+            module = _load_module(
+                self._evaluator,
+                import_names=import_parts,
+                sys_path=sys_path,
+            )
+            return ContextSet(module)
+
         module_name = '.'.join(import_parts)
         try:
-            return set([self._evaluator.modules[module_name]])
+            return ContextSet(self._evaluator.module_cache.get(module_name))
         except KeyError:
             pass
 
@@ -305,7 +323,7 @@ class Importer(object):
             # the module cache.
             bases = self._do_import(import_path[:-1], sys_path)
             if not bases:
-                return set()
+                return NO_CONTEXTS
             # We can take the first element, because only the os special
             # case yields multiple modules, which is not important for
             # further imports.
@@ -323,70 +341,53 @@ class Importer(object):
             except AttributeError:
                 # The module is not a package.
                 _add_error(self.module_context, import_path[-1])
-                return set()
+                return NO_CONTEXTS
             else:
                 paths = method()
                 debug.dbg('search_module %s in paths %s', module_name, paths)
                 for path in paths:
                     # At the moment we are only using one path. So this is
                     # not important to be correct.
-                    try:
-                        if not isinstance(path, list):
-                            path = [path]
-                        module_file, module_path, is_pkg = \
-                            find_module(import_parts[-1], path, fullname=module_name)
+                    if not isinstance(path, list):
+                        path = [path]
+                    code, module_path, is_pkg = self._evaluator.compiled_subprocess.get_module_info(
+                        string=import_parts[-1],
+                        path=path,
+                        full_name=module_name,
+                        is_global_search=False,
+                    )
+                    if module_path is not None:
                         break
-                    except ImportError:
-                        module_path = None
-                if module_path is None:
+                else:
                     _add_error(self.module_context, import_path[-1])
-                    return set()
+                    return NO_CONTEXTS
         else:
-            parent_module = None
-            try:
-                debug.dbg('search_module %s in %s', import_parts[-1], self.file_path)
-                # Override the sys.path. It works only good that way.
-                # Injecting the path directly into `find_module` did not work.
-                sys.path, temp = sys_path, sys.path
-                try:
-                    module_file, module_path, is_pkg = \
-                        find_module(import_parts[-1], fullname=module_name)
-                finally:
-                    sys.path = temp
-            except ImportError:
+            debug.dbg('global search_module %s in %s', import_parts[-1], self.file_path)
+            # Override the sys.path. It works only good that way.
+            # Injecting the path directly into `find_module` did not work.
+            code, module_path, is_pkg = self._evaluator.compiled_subprocess.get_module_info(
+                string=import_parts[-1],
+                full_name=module_name,
+                sys_path=sys_path,
+                is_global_search=True,
+            )
+            if module_path is None:
                 # The module is not a package.
                 _add_error(self.module_context, import_path[-1])
-                return set()
+                return NO_CONTEXTS
 
-        code = None
-        if is_pkg:
-            # In this case, we don't have a file yet. Search for the
-            # __init__ file.
-            if module_path.endswith(('.zip', '.egg')):
-                code = module_file.loader.get_source(module_name)
-            else:
-                module_path = get_init_path(module_path)
-        elif module_file:
-            code = module_file.read()
-            module_file.close()
-
-        if isinstance(module_path, ImplicitNSInfo):
-            from jedi.evaluate.representation import ImplicitNamespaceContext
-            fullname, paths = module_path.name, module_path.paths
-            module = ImplicitNamespaceContext(self._evaluator, fullname=fullname)
-            module.paths = paths
-        elif module_file is None and not module_path.endswith(('.py', '.zip', '.egg')):
-            module = compiled.load_module(self._evaluator, module_path)
-        else:
-            module = _load_module(self._evaluator, module_path, code, sys_path, parent_module)
+        module = _load_module(
+            self._evaluator, module_path, code, sys_path,
+            import_names=import_parts,
+            safe_module_name=True,
+        )
 
         if module is None:
             # The file might raise an ImportError e.g. and therefore not be
             # importable.
-            return set()
+            return NO_CONTEXTS
 
-        self._evaluator.modules[module_name] = module
-        return set([module])
+        return ContextSet(module)
 
     def _generate_name(self, name, in_module=None):
         # Create a pseudo import to be able to follow them.
@@ -399,15 +400,17 @@ class Importer(object):
         Get the names of all modules in the search_path. This means file names
         and not names defined in the files.
         """
+        sub = self._evaluator.compiled_subprocess
 
         names = []
         # add builtin module names
         if search_path is None and in_module is None:
-            names += [self._generate_name(name) for name in sys.builtin_module_names]
+            names += [self._generate_name(name) for name in sub.get_builtin_module_names()]
 
         if search_path is None:
             search_path = self.sys_path_with_modifications()
-        for module_loader, name, is_pkg in pkgutil.iter_modules(search_path):
+
+        for name in sub.list_module_names(search_path):
             names.append(self._generate_name(name, in_module=in_module))
         return names
 
@@ -416,7 +419,8 @@ class Importer(object):
         :param only_modules: Indicates wheter it's possible to import a
             definition that is not defined in a module.
         """
-        from jedi.evaluate.representation import ModuleContext, ImplicitNamespaceContext
+        from jedi.evaluate.context import ModuleContext
+        from jedi.evaluate.context.namespace import ImplicitNamespaceContext
         names = []
         if self.import_path:
             # flask
@@ -445,7 +449,7 @@ class Importer(object):
                 # implicit namespace packages
                 elif isinstance(context, ImplicitNamespaceContext):
                     paths = context.paths
-                    names += self._get_module_names(paths)
+                    names += self._get_module_names(paths, in_module=context)
 
                 if only_modules:
                     # In the case of an import like `from x.` we don't need to
@@ -473,61 +477,94 @@ class Importer(object):
         return names
 
 
-def _load_module(evaluator, path=None, code=None, sys_path=None, parent_module=None):
-    if sys_path is None:
-        sys_path = evaluator.sys_path
-
-    dotted_path = path and compiled.dotted_from_fs_path(path, sys_path)
-    if path is not None and path.endswith(('.py', '.zip', '.egg')) \
-            and dotted_path not in settings.auto_import_modules:
-
-        module_node = evaluator.grammar.parse(
-            code=code, path=path, cache=True, diff_cache=True,
-            cache_path=settings.cache_directory)
-
-        from jedi.evaluate.representation import ModuleContext
-        return ModuleContext(evaluator, module_node, path=path)
+def _load_module(evaluator, path=None, code=None, sys_path=None,
+                 import_names=None, safe_module_name=False):
+    if import_names is None:
+        dotted_name = None
     else:
-        return compiled.load_module(evaluator, path)
+        dotted_name = '.'.join(import_names)
+    try:
+        return evaluator.module_cache.get(dotted_name)
+    except KeyError:
+        pass
+    try:
+        return evaluator.module_cache.get_from_path(path)
+    except KeyError:
+        pass
+
+    if isinstance(path, ImplicitNSInfo):
+        from jedi.evaluate.context.namespace import ImplicitNamespaceContext
+        module = ImplicitNamespaceContext(
+            evaluator,
+            fullname=path.name,
+            paths=path.paths,
+        )
+    else:
+        if sys_path is None:
+            sys_path = evaluator.get_sys_path()
+
+        if path is not None and path.endswith(('.py', '.zip', '.egg')):
+            module_node = evaluator.parse(
+                code=code, path=path, cache=True,
+                diff_cache=settings.fast_parser,
+                cache_path=settings.cache_directory)
+
+            from jedi.evaluate.context import ModuleContext
+            module = ModuleContext(
+                evaluator, module_node,
+                path=path,
+                code_lines=get_cached_code_lines(evaluator.grammar, path),
+            )
+        else:
+            assert dotted_name is not None
+            module = compiled.load_module(evaluator, dotted_name=dotted_name, sys_path=sys_path)
+
+    if module is not None and dotted_name is not None:
+        add_module_to_cache(evaluator, dotted_name, module, safe=safe_module_name)
+
+    return module
 
 
-def add_module(evaluator, module_name, module):
-    if '.' not in module_name:
+def add_module_to_cache(evaluator, module_name, module, safe=False):
+    if not safe and '.' not in module_name:
         # We cannot add paths with dots, because that would collide with
         # the sepatator dots for nested packages. Therefore we return
         # `__main__` in ModuleWrapper.py__name__(), which is similar to
         # Python behavior.
-        evaluator.modules[module_name] = module
+        return
+    evaluator.module_cache.add(module, module_name)
 
 
 def get_modules_containing_name(evaluator, modules, name):
     """
     Search a name in the directories of modules.
     """
-    from jedi.evaluate import representation as er
-
-    def check_python_file(path):
-        try:
-            # TODO I don't think we should use the cache here?!
-            node_cache_item = parser_cache[evaluator.grammar._hashed][path]
-        except KeyError:
-            try:
-                return check_fs(path)
-            except IOError:
-                return None
-        else:
-            module_node = node_cache_item.node
-            return er.ModuleContext(evaluator, module_node, path=path)
+    def check_directories(paths):
+        for p in paths:
+            if p is not None:
+                # We need abspath, because the seetings paths might not already
+                # have been converted to absolute paths.
+                d = os.path.dirname(os.path.abspath(p))
+                for file_name in os.listdir(d):
+                    path = os.path.join(d, file_name)
+                    if file_name.endswith('.py'):
+                        yield path
 
     def check_fs(path):
-        with open(path, 'rb') as f:
+        try:
+            f = open(path, 'rb')
+        except FileNotFoundError:
+            return
+        with f:
             code = python_bytes_to_unicode(f.read(), errors='replace')
             if name in code:
-                module = _load_module(evaluator, path, code)
-
-                module_name = sys_path.dotted_path_in_sys_path(evaluator.sys_path, path)
-                if module_name is not None:
-                    add_module(evaluator, module_name, module)
+                e_sys_path = evaluator.get_sys_path()
+                import_names = sys_path.dotted_path_in_sys_path(e_sys_path, path)
+                module = _load_module(
+                    evaluator, path, code,
+                    sys_path=e_sys_path,
+                    import_names=import_names,
+                )
                 return module
 
     # skip non python modules
@@ -544,21 +581,14 @@ def get_modules_containing_name(evaluator, modules, name):
     if not settings.dynamic_params_for_other_modules:
         return
 
-    paths = set(settings.additional_dynamic_modules)
-    for p in used_mod_paths:
-        if p is not None:
-            # We need abspath, because the seetings paths might not already
-            # have been converted to absolute paths.
-            d = os.path.dirname(os.path.abspath(p))
-            for file_name in os.listdir(d):
-                path = os.path.join(d, file_name)
-                if path not in used_mod_paths and path not in paths:
-                    if file_name.endswith('.py'):
-                        paths.add(path)
+    additional = set(os.path.abspath(p) for p in settings.additional_dynamic_modules)
+    # Check the directories of used modules.
+    paths = (additional | set(check_directories(used_mod_paths))) \
+            - used_mod_paths
 
     # Sort here to make issues less random.
     for p in sorted(paths):
         # make testing easier, sort it - same results on every interpreter
-        m = check_python_file(p)
+        m = check_fs(p)
         if m is not None and not isinstance(m, compiled.CompiledObject):
             yield m

jedi/evaluate/jedi_typing.py

@@ -81,6 +81,9 @@ def factory(typing_name, indextypes):
     class Dict(MutableMapping, dict):
         pass
 
+    class DefaultDict(MutableMapping, dict):
+        pass
+
     dct = {
         "Sequence": Sequence,
         "MutableSequence": MutableSequence,
@@ -96,5 +99,6 @@ def factory(typing_name, indextypes):
         "ItemsView": ItemsView,
         "ValuesView": ValuesView,
         "Dict": Dict,
+        "DefaultDict": DefaultDict,
     }
     return dct[typing_name]

jedi/evaluate/lazy_context.py

@@ -0,0 +1,59 @@
+from jedi.evaluate.base_context import ContextSet, NO_CONTEXTS
+from jedi.common.utils import monkeypatch
+
+
+class AbstractLazyContext(object):
+    def __init__(self, data):
+        self.data = data
+
+    def __repr__(self):
+        return '<%s: %s>' % (self.__class__.__name__, self.data)
+
+    def infer(self):
+        raise NotImplementedError
+
+
+class LazyKnownContext(AbstractLazyContext):
+    """data is a context."""
+    def infer(self):
+        return ContextSet(self.data)
+
+
+class LazyKnownContexts(AbstractLazyContext):
+    """data is a ContextSet."""
+    def infer(self):
+        return self.data
+
+
+class LazyUnknownContext(AbstractLazyContext):
+    def __init__(self):
+        super(LazyUnknownContext, self).__init__(None)
+
+    def infer(self):
+        return NO_CONTEXTS
+
+
+class LazyTreeContext(AbstractLazyContext):
+    def __init__(self, context, node):
+        super(LazyTreeContext, self).__init__(node)
+        self._context = context
+        # We need to save the predefined names. It's an unfortunate side effect
+        # that needs to be tracked otherwise results will be wrong.
+        self._predefined_names = dict(context.predefined_names)
+
+    def infer(self):
+        with monkeypatch(self._context, 'predefined_names', self._predefined_names):
+            return self._context.eval_node(self.data)
+
+
+def get_merged_lazy_context(lazy_contexts):
+    if len(lazy_contexts) > 1:
+        return MergedLazyContexts(lazy_contexts)
+    else:
+        return lazy_contexts[0]
+
+
+class MergedLazyContexts(AbstractLazyContext):
+    """data is a list of lazy contexts."""
+    def infer(self):
+        return ContextSet.from_sets(l.infer() for l in self.data)

jedi/evaluate/param.py

@@ -1,230 +1,22 @@
 from collections import defaultdict
 
-from jedi._compatibility import zip_longest
-from jedi import debug
-from jedi import common
-from parso.python import tree
-from jedi.evaluate import iterable
+from jedi.evaluate.utils import PushBackIterator
 from jedi.evaluate import analysis
-from jedi.evaluate import context
+from jedi.evaluate.lazy_context import LazyKnownContext, \
+    LazyTreeContext, LazyUnknownContext
 from jedi.evaluate import docstrings
 from jedi.evaluate import pep0484
-from jedi.evaluate.filters import ParamName
+from jedi.evaluate.context import iterable
 
 
-def add_argument_issue(parent_context, error_name, lazy_context, message):
-    if isinstance(lazy_context, context.LazyTreeContext):
+def _add_argument_issue(parent_context, error_name, lazy_context, message):
+    if isinstance(lazy_context, LazyTreeContext):
         node = lazy_context.data
         if node.parent.type == 'argument':
             node = node.parent
         analysis.add(parent_context, error_name, node, message)
 
 
-def try_iter_content(types, depth=0):
-    """Helper method for static analysis."""
-    if depth > 10:
-        # It's possible that a loop has references on itself (especially with
-        # CompiledObject). Therefore don't loop infinitely.
-        return
-
-    for typ in types:
-        try:
-            f = typ.py__iter__
-        except AttributeError:
-            pass
-        else:
-            for lazy_context in f():
-                try_iter_content(lazy_context.infer(), depth + 1)
-
-
-class AbstractArguments():
-    context = None
-
-    def eval_argument_clinic(self, parameters):
-        """Uses a list with argument clinic information (see PEP 436)."""
-        iterator = self.unpack()
-        for i, (name, optional, allow_kwargs) in enumerate(parameters):
-            key, argument = next(iterator, (None, None))
-            if key is not None:
-                raise NotImplementedError
-            if argument is None and not optional:
-                debug.warning('TypeError: %s expected at least %s arguments, got %s',
-                              name, len(parameters), i)
-                raise ValueError
-            values = set() if argument is None else argument.infer()
-
-            if not values and not optional:
-                # For the stdlib we always want values. If we don't get them,
-                # that's ok, maybe something is too hard to resolve, however,
-                # we will not proceed with the evaluation of that function.
-                debug.warning('argument_clinic "%s" not resolvable.', name)
-                raise ValueError
-            yield values
-
-    def eval_all(self, funcdef=None):
-        """
-        Evaluates all arguments as a support for static analysis
-        (normally Jedi).
-        """
-        for key, lazy_context in self.unpack():
-            types = lazy_context.infer()
-            try_iter_content(types)
-
-    def get_calling_nodes(self):
-        raise NotImplementedError
-
-    def unpack(self, funcdef=None):
-        raise NotImplementedError
-
-    def get_params(self, execution_context):
-        return get_params(execution_context, self)
-
-
-class AnonymousArguments(AbstractArguments):
-    def get_params(self, execution_context):
-        from jedi.evaluate.dynamic import search_params
-        return search_params(
-            execution_context.evaluator,
-            execution_context,
-            execution_context.tree_node
-        )
-
-
-class TreeArguments(AbstractArguments):
-    def __init__(self, evaluator, context, argument_node, trailer=None):
-        """
-        The argument_node is either a parser node or a list of evaluated
-        objects. Those evaluated objects may be lists of evaluated objects
-        themselves (one list for the first argument, one for the second, etc).
-
-        :param argument_node: May be an argument_node or a list of nodes.
-        """
-        self.argument_node = argument_node
-        self.context = context
-        self._evaluator = evaluator
-        self.trailer = trailer  # Can be None, e.g. in a class definition.
-
-    def _split(self):
-        if isinstance(self.argument_node, (tuple, list)):
-            for el in self.argument_node:
-                yield 0, el
-        else:
-            if not (self.argument_node.type == 'arglist' or (
-                    # in python 3.5 **arg is an argument, not arglist
-                    (self.argument_node.type == 'argument') and
-                     self.argument_node.children[0] in ('*', '**'))):
-                yield 0, self.argument_node
-                return
-
-            iterator = iter(self.argument_node.children)
-            for child in iterator:
-                if child == ',':
-                    continue
-                elif child in ('*', '**'):
-                    yield len(child.value), next(iterator)
-                elif child.type == 'argument' and \
-                        child.children[0] in ('*', '**'):
-                    assert len(child.children) == 2
-                    yield len(child.children[0].value), child.children[1]
-                else:
-                    yield 0, child
-
-    def unpack(self, funcdef=None):
-        named_args = []
-        for star_count, el in self._split():
-            if star_count == 1:
-                arrays = self.context.eval_node(el)
-                iterators = [_iterate_star_args(self.context, a, el, funcdef)
-                             for a in arrays]
-                iterators = list(iterators)
-                for values in list(zip_longest(*iterators)):
-                    # TODO zip_longest yields None, that means this would raise
-                    # an exception?
-                    yield None, context.get_merged_lazy_context(
-                        [v for v in values if v is not None]
-                    )
-            elif star_count == 2:
-                arrays = self._evaluator.eval_element(self.context, el)
-                for dct in arrays:
-                    for key, values in _star_star_dict(self.context, dct, el, funcdef):
-                        yield key, values
-            else:
-                if el.type == 'argument':
-                    c = el.children
-                    if len(c) == 3:  # Keyword argument.
-                        named_args.append((c[0].value, context.LazyTreeContext(self.context, c[2]),))
-                    else:  # Generator comprehension.
-                        # Include the brackets with the parent.
-                        comp = iterable.GeneratorComprehension(
-                            self._evaluator, self.context, self.argument_node.parent)
-                        yield None, context.LazyKnownContext(comp)
-                else:
-                    yield None, context.LazyTreeContext(self.context, el)
-
-        # Reordering var_args is necessary, because star args sometimes appear
-        # after named argument, but in the actual order it's prepended.
-        for named_arg in named_args:
-            yield named_arg
-
-    def as_tree_tuple_objects(self):
-        for star_count, argument in self._split():
-            if argument.type == 'argument':
-                argument, default = argument.children[::2]
-            else:
-                default = None
-            yield argument, default, star_count
-
-    def __repr__(self):
-        return '<%s: %s>' % (self.__class__.__name__, self.argument_node)
-
-    def get_calling_nodes(self):
-        from jedi.evaluate.dynamic import MergedExecutedParams
-        old_arguments_list = []
-        arguments = self
-
-        while arguments not in old_arguments_list:
-            if not isinstance(arguments, TreeArguments):
-                break
-
-            old_arguments_list.append(arguments)
-            for name, default, star_count in reversed(list(arguments.as_tree_tuple_objects())):
-                if not star_count or not isinstance(name, tree.Name):
-                    continue
-
-                names = self._evaluator.goto(arguments.context, name)
-                if len(names) != 1:
-                    break
-                if not isinstance(names[0], ParamName):
-                    break
-                param = names[0].get_param()
-                if isinstance(param, MergedExecutedParams):
-                    # For dynamic searches we don't even want to see errors.
-                    return []
-                if not isinstance(param, ExecutedParam):
-                    break
-                if param.var_args is None:
-                    break
-                arguments = param.var_args
-                break
-
-        return [arguments.argument_node or arguments.trailer]
-
-
-class ValuesArguments(AbstractArguments):
-    def __init__(self, values_list):
-        self._values_list = values_list
-
-    def unpack(self, funcdef=None):
-        for values in self._values_list:
-            yield None, context.LazyKnownContexts(values)
-
-    def get_calling_nodes(self):
-        return []
-
-    def __repr__(self):
-        return '<%s: %s>' % (self.__class__.__name__, self._values_list)
-
-
 class ExecutedParam(object):
     """Fake a param and give it values."""
     def __init__(self, execution_context, param_node, lazy_context):
@@ -237,7 +29,7 @@ class ExecutedParam(object):
         pep0484_hints = pep0484.infer_param(self._execution_context, self._param_node)
         doc_params = docstrings.infer_param(self._execution_context, self._param_node)
         if pep0484_hints or doc_params:
-            return list(set(pep0484_hints) | set(doc_params))
+            return pep0484_hints | doc_params
 
         return self._lazy_context.infer()
 
@@ -249,7 +41,7 @@ class ExecutedParam(object):
         return '<%s: %s>' % (self.__class__.__name__, self.string_name)
 
 
-def get_params(execution_context, var_args):
+def get_executed_params(execution_context, var_args):
     result_params = []
     param_dict = {}
     funcdef = execution_context.tree_node
@@ -258,7 +50,7 @@ def get_params(execution_context, var_args):
     for param in funcdef.get_params():
         param_dict[param.name.value] = param
     unpacked_va = list(var_args.unpack(funcdef))
-    var_arg_iterator = common.PushBackIterator(iter(unpacked_va))
+    var_arg_iterator = PushBackIterator(iter(unpacked_va))
 
     non_matching_keys = defaultdict(lambda: [])
     keys_used = {}
@@ -305,31 +97,31 @@ def get_params(execution_context, var_args):
                         var_arg_iterator.push_back((key, argument))
                         break
                     lazy_context_list.append(argument)
-            seq = iterable.FakeSequence(execution_context.evaluator, 'tuple', lazy_context_list)
-            result_arg = context.LazyKnownContext(seq)
+            seq = iterable.FakeSequence(execution_context.evaluator, u'tuple', lazy_context_list)
+            result_arg = LazyKnownContext(seq)
         elif param.star_count == 2:
             # **kwargs param
             dct = iterable.FakeDict(execution_context.evaluator, dict(non_matching_keys))
-            result_arg = context.LazyKnownContext(dct)
+            result_arg = LazyKnownContext(dct)
             non_matching_keys = {}
         else:
             # normal param
             if argument is None:
                 # No value: Return an empty container
                 if param.default is None:
-                    result_arg = context.LazyUnknownContext()
+                    result_arg = LazyUnknownContext()
                     if not keys_only:
                         for node in var_args.get_calling_nodes():
                             m = _error_argument_count(funcdef, len(unpacked_va))
                             analysis.add(parent_context, 'type-error-too-few-arguments',
                                          node, message=m)
                 else:
-                    result_arg = context.LazyTreeContext(parent_context, param.default)
+                    result_arg = LazyTreeContext(parent_context, param.default)
             else:
                 result_arg = argument
 
         result_params.append(ExecutedParam(execution_context, param, result_arg))
-        if not isinstance(result_arg, context.LazyUnknownContext):
+        if not isinstance(result_arg, LazyUnknownContext):
             keys_used[param.name.value] = result_params[-1]
 
     if keys_only:
@@ -350,7 +142,7 @@ def get_params(execution_context, var_args):
     for key, lazy_context in non_matching_keys.items():
         m = "TypeError: %s() got an unexpected keyword argument '%s'." \
             % (funcdef.name, key)
-        add_argument_issue(
+        _add_argument_issue(
             parent_context,
             'type-error-keyword-argument',
             lazy_context,
@@ -365,40 +157,10 @@ def get_params(execution_context, var_args):
         first_key, lazy_context = remaining_arguments[0]
         if var_args.get_calling_nodes():
             # There might not be a valid calling node so check for that first.
-            add_argument_issue(parent_context, 'type-error-too-many-arguments', lazy_context, message=m)
+            _add_argument_issue(parent_context, 'type-error-too-many-arguments', lazy_context, message=m)
     return result_params
 
 
-def _iterate_star_args(context, array, input_node, funcdef=None):
-    try:
-        iter_ = array.py__iter__
-    except AttributeError:
-        if funcdef is not None:
-            # TODO this funcdef should not be needed.
-            m = "TypeError: %s() argument after * must be a sequence, not %s" \
-                % (funcdef.name.value, array)
-            analysis.add(context, 'type-error-star', input_node, message=m)
-    else:
-        for lazy_context in iter_():
-            yield lazy_context
-
-
-def _star_star_dict(context, array, input_node, funcdef):
-    from jedi.evaluate.instance import CompiledInstance
-    if isinstance(array, CompiledInstance) and array.name.string_name == 'dict':
-        # For now ignore this case. In the future add proper iterators and just
-        # make one call without crazy isinstance checks.
-        return {}
-    elif isinstance(array, iterable.AbstractSequence) and array.array_type == 'dict':
-        return array.exact_key_items()
-    else:
-        if funcdef is not None:
-            m = "TypeError: %s argument after ** must be a mapping, not %s" \
-                % (funcdef.name.value, array)
-            analysis.add(context, 'type-error-star-star', input_node, message=m)
-        return {}
-
-
 def _error_argument_count(funcdef, actual_count):
     params = funcdef.get_params()
     default_arguments = sum(1 for p in params if p.default or p.star_count)
@@ -413,21 +175,20 @@ def _error_argument_count(funcdef, actual_count):
 
 def _create_default_param(execution_context, param):
     if param.star_count == 1:
-        result_arg = context.LazyKnownContext(
-            iterable.FakeSequence(execution_context.evaluator, 'tuple', [])
+        result_arg = LazyKnownContext(
+            iterable.FakeSequence(execution_context.evaluator, u'tuple', [])
         )
     elif param.star_count == 2:
-        result_arg = context.LazyKnownContext(
+        result_arg = LazyKnownContext(
             iterable.FakeDict(execution_context.evaluator, {})
         )
     elif param.default is None:
-        result_arg = context.LazyUnknownContext()
+        result_arg = LazyUnknownContext()
     else:
-        result_arg = context.LazyTreeContext(execution_context.parent_context, param.default)
+        result_arg = LazyTreeContext(execution_context.parent_context, param.default)
     return ExecutedParam(execution_context, param, result_arg)
 
 
 def create_default_params(execution_context, funcdef):
     return [_create_default_param(execution_context, p)
             for p in funcdef.get_params()]
-

jedi/evaluate/pep0484.py

@@ -19,19 +19,20 @@ x support for type hint comments for functions, `# type: (int, str) -> int`.
     See comment from Guido https://github.com/davidhalter/jedi/issues/662
 """
 
-import itertools
 import os
 import re
 
-from parso import ParserSyntaxError
+from parso import ParserSyntaxError, parse, split_lines
 from parso.python import tree
 
-from jedi.common import unite
+from jedi._compatibility import unicode, force_unicode
 from jedi.evaluate.cache import evaluator_method_cache
 from jedi.evaluate import compiled
-from jedi.evaluate.context import LazyTreeContext
+from jedi.evaluate.base_context import NO_CONTEXTS, ContextSet
+from jedi.evaluate.lazy_context import LazyTreeContext
+from jedi.evaluate.context import ModuleContext
+from jedi.evaluate.helpers import is_string
 from jedi import debug
-from jedi import _compatibility
 from jedi import parser_utils
 
 
@@ -41,17 +42,22 @@ def _evaluate_for_annotation(context, annotation, index=None):
     If index is not None, the annotation is expected to be a tuple
     and we're interested in that index
     """
-    if annotation is not None:
-        definitions = context.eval_node(
-            _fix_forward_reference(context, annotation))
-        if index is not None:
-            definitions = list(itertools.chain.from_iterable(
-                definition.py__getitem__(index) for definition in definitions
-                if definition.array_type == 'tuple' and
-                len(list(definition.py__iter__())) >= index))
-        return unite(d.execute_evaluated() for d in definitions)
-    else:
-        return set()
+    context_set = context.eval_node(_fix_forward_reference(context, annotation))
+    return context_set.execute_evaluated()
+
+
+def _evaluate_annotation_string(context, string, index=None):
+    node = _get_forward_reference_node(context, string)
+    if node is None:
+        return NO_CONTEXTS
+
+    context_set = context.eval_node(node)
+    if index is not None:
+        context_set = context_set.filter(
+            lambda context: context.array_type == u'tuple'
+                            and len(list(context.py__iter__())) >= index
+        ).py__getitem__(index)
+    return context_set.execute_evaluated()
 
 
 def _fix_forward_reference(context, node):
@@ -60,30 +66,111 @@ def _fix_forward_reference(context, node):
         debug.warning("Eval'ed typing index %s should lead to 1 object, "
                       " not %s" % (node, evaled_nodes))
         return node
-    evaled_node = list(evaled_nodes)[0]
-    if isinstance(evaled_node, compiled.CompiledObject) and \
-            isinstance(evaled_node.obj, str):
-        try:
-            new_node = context.evaluator.grammar.parse(
-                _compatibility.unicode(evaled_node.obj),
-                start_symbol='eval_input',
-                error_recovery=False
-            )
-        except ParserSyntaxError:
-            debug.warning('Annotation not parsed: %s' % evaled_node.obj)
-            return node
-        else:
-            module = node.get_root_node()
-            parser_utils.move(new_node, module.end_pos[0])
-            new_node.parent = context.tree_node
-            return new_node
+
+    evaled_context = list(evaled_nodes)[0]
+    if is_string(evaled_context):
+        result = _get_forward_reference_node(context, evaled_context.get_safe_value())
+        if result is not None:
+            return result
+
+    return node
+
+
+def _get_forward_reference_node(context, string):
+    try:
+        new_node = context.evaluator.grammar.parse(
+            force_unicode(string),
+            start_symbol='eval_input',
+            error_recovery=False
+        )
+    except ParserSyntaxError:
+        debug.warning('Annotation not parsed: %s' % string)
+        return None
     else:
-        return node
+        module = context.tree_node.get_root_node()
+        parser_utils.move(new_node, module.end_pos[0])
+        new_node.parent = context.tree_node
+        return new_node
+
+
+def _split_comment_param_declaration(decl_text):
+    """
+    Split decl_text on commas, but group generic expressions
+    together.
+
+    For example, given "foo, Bar[baz, biz]" we return
+    ['foo', 'Bar[baz, biz]'].
+
+    """
+    try:
+        node = parse(decl_text, error_recovery=False).children[0]
+    except ParserSyntaxError:
+        debug.warning('Comment annotation is not valid Python: %s' % decl_text)
+        return []
+
+    if node.type == 'name':
+        return [node.get_code().strip()]
+
+    params = []
+    try:
+        children = node.children
+    except AttributeError:
+        return []
+    else:
+        for child in children:
+            if child.type in ['name', 'atom_expr', 'power']:
+                params.append(child.get_code().strip())
+
+    return params
 
 
 @evaluator_method_cache()
 def infer_param(execution_context, param):
+    """
+    Infers the type of a function parameter, using type annotations.
+    """
     annotation = param.annotation
+    if annotation is None:
+        # If no Python 3-style annotation, look for a Python 2-style comment
+        # annotation.
+        # Identify parameters to function in the same sequence as they would
+        # appear in a type comment.
+        all_params = [child for child in param.parent.children
+                      if child.type == 'param']
+
+        node = param.parent.parent
+        comment = parser_utils.get_following_comment_same_line(node)
+        if comment is None:
+            return NO_CONTEXTS
+
+        match = re.match(r"^#\s*type:\s*\(([^#]*)\)\s*->", comment)
+        if not match:
+            return NO_CONTEXTS
+        params_comments = _split_comment_param_declaration(match.group(1))
+
+        # Find the specific param being investigated
+        index = all_params.index(param)
+        # If the number of parameters doesn't match length of type comment,
+        # ignore first parameter (assume it's self).
+        if len(params_comments) != len(all_params):
+            debug.warning(
+                "Comments length != Params length %s %s",
+                params_comments, all_params
+            )
+        from jedi.evaluate.context.instance import InstanceArguments
+        if isinstance(execution_context.var_args, InstanceArguments):
+            if index == 0:
+                # Assume it's self, which is already handled
+                return NO_CONTEXTS
+            index -= 1
+        if index >= len(params_comments):
+            return NO_CONTEXTS
+
+        param_comment = params_comments[index]
+        return _evaluate_annotation_string(
+            execution_context.get_root_context(),
+            param_comment
+        )
     module_context = execution_context.get_root_context()
     return _evaluate_for_annotation(module_context, annotation)
 
@@ -103,12 +190,33 @@ def py__annotations__(funcdef):
 
 @evaluator_method_cache()
 def infer_return_types(function_context):
+    """
+    Infers the type of a function's return value,
+    according to type annotations.
+    """
     annotation = py__annotations__(function_context.tree_node).get("return", None)
+    if annotation is None:
+        # If there is no Python 3-type annotation, look for a Python 2-type annotation
+        node = function_context.tree_node
+        comment = parser_utils.get_following_comment_same_line(node)
+        if comment is None:
+            return NO_CONTEXTS
+
+        match = re.match(r"^#\s*type:\s*\([^#]*\)\s*->\s*([^#]*)", comment)
+        if not match:
+            return NO_CONTEXTS
+
+        return _evaluate_annotation_string(
+            function_context.get_root_context(),
+            match.group(1).strip()
+        )
+
     module_context = function_context.get_root_context()
     return _evaluate_for_annotation(module_context, annotation)
 
 
 _typing_module = None
+_typing_module_code_lines = None
 
 
 def _get_typing_replacement_module(grammar):
@@ -116,14 +224,15 @@ def _get_typing_replacement_module(grammar):
     The idea is to return our jedi replacement for the PEP-0484 typing module
     as discussed at https://github.com/davidhalter/jedi/issues/663
     """
-    global _typing_module
+    global _typing_module, _typing_module_code_lines
     if _typing_module is None:
         typing_path = \
             os.path.abspath(os.path.join(__file__, "../jedi_typing.py"))
         with open(typing_path) as f:
-            code = _compatibility.unicode(f.read())
+            code = unicode(f.read())
         _typing_module = grammar.parse(code)
-    return _typing_module
+        _typing_module_code_lines = split_lines(code, keepends=True)
+    return _typing_module, _typing_module_code_lines
 
 
 def py__getitem__(context, typ, node):
@@ -147,17 +256,18 @@ def py__getitem__(context, typ, node):
     if type_name in ("Union", '_Union'):
         # In Python 3.6 it's still called typing.Union but it's an instance
         # called _Union.
-        return unite(context.eval_node(node) for node in nodes)
+        return ContextSet.from_sets(context.eval_node(node) for node in nodes)
     if type_name in ("Optional", '_Optional'):
         # Here we have the same issue like in Union. Therefore we also need to
         # check for the instance typing._Optional (Python 3.6).
         return context.eval_node(nodes[0])
 
-    from jedi.evaluate.representation import ModuleContext
+    module_node, code_lines = _get_typing_replacement_module(context.evaluator.latest_grammar)
     typing = ModuleContext(
         context.evaluator,
-        module_node=_get_typing_replacement_module(context.evaluator.latest_grammar),
-        path=None
+        module_node=module_node,
+        path=None,
+        code_lines=code_lines,
     )
     factories = typing.py__getattribute__("factory")
     assert len(factories) == 1
@@ -169,12 +279,12 @@ def py__getitem__(context, typ, node):
                            if isinstance(child, tree.Class))
     if type_name not in valid_classnames:
         return None
-    compiled_classname = compiled.create(context.evaluator, type_name)
+    compiled_classname = compiled.create_simple_object(context.evaluator, type_name)
 
-    from jedi.evaluate.iterable import FakeSequence
+    from jedi.evaluate.context.iterable import FakeSequence
     args = FakeSequence(
         context.evaluator,
-        "tuple",
+        u'tuple',
         [LazyTreeContext(context, n) for n in nodes]
     )
 
@@ -215,10 +325,6 @@ def _find_type_from_comment_hint(context, node, varlist, name):
     if comment is None:
         return []
     match = re.match(r"^#\s*type:\s*([^#]*)", comment)
-    if not match:
+    if match is None:
         return []
-    annotation = tree.String(
-        repr(str(match.group(1).strip())),
-        node.start_pos)
-    annotation.parent = node.parent
-    return _evaluate_for_annotation(context, annotation, index)
+    return _evaluate_annotation_string(context, match.group(1).strip(), index)

jedi/evaluate/precedence.py

@@ -1,178 +0,0 @@
-"""
-Handles operator precedence.
-"""
-import operator as op
-
-from jedi._compatibility import unicode
-from jedi import debug
-from jedi.evaluate.compiled import CompiledObject, create, builtin_from_name
-from jedi.evaluate import analysis
-
-# Maps Python syntax to the operator module.
-COMPARISON_OPERATORS = {
-    '==': op.eq,
-    '!=': op.ne,
-    'is': op.is_,
-    'is not': op.is_not,
-    '<': op.lt,
-    '<=': op.le,
-    '>': op.gt,
-    '>=': op.ge,
-}
-
-
-def literals_to_types(evaluator, result):
-    # Changes literals ('a', 1, 1.0, etc) to its type instances (str(),
-    # int(), float(), etc).
-    new_result = set()
-    for typ in result:
-        if is_literal(typ):
-            # Literals are only valid as long as the operations are
-            # correct. Otherwise add a value-free instance.
-            cls = builtin_from_name(evaluator, typ.name.string_name)
-            new_result |= cls.execute_evaluated()
-        else:
-            new_result.add(typ)
-    return new_result
-
-
-def calculate_children(evaluator, context, children):
-    """
-    Calculate a list of children with operators.
-    """
-    iterator = iter(children)
-    types = context.eval_node(next(iterator))
-    for operator in iterator:
-        right = next(iterator)
-        if operator.type == 'comp_op':  # not in / is not
-            operator = ' '.join(c.value for c in operator.children)
-
-        # handle lazy evaluation of and/or here.
-        if operator in ('and', 'or'):
-            left_bools = set([left.py__bool__() for left in types])
-            if left_bools == set([True]):
-                if operator == 'and':
-                    types = context.eval_node(right)
-            elif left_bools == set([False]):
-                if operator != 'and':
-                    types = context.eval_node(right)
-            # Otherwise continue, because of uncertainty.
-        else:
-            types = calculate(evaluator, context, types, operator,
-                              context.eval_node(right))
-    debug.dbg('calculate_children types %s', types)
-    return types
-
-
-def calculate(evaluator, context, left_result, operator, right_result):
-    result = set()
-    if not left_result or not right_result:
-        # illegal slices e.g. cause left/right_result to be None
-        result = (left_result or set()) | (right_result or set())
-        result = literals_to_types(evaluator, result)
-    else:
-        # I don't think there's a reasonable chance that a string
-        # operation is still correct, once we pass something like six
-        # objects.
-        if len(left_result) * len(right_result) > 6:
-            result = literals_to_types(evaluator, left_result | right_result)
-        else:
-            for left in left_result:
-                for right in right_result:
-                    result |= _element_calculate(evaluator, context, left, operator, right)
-    return result
-
-
-def factor_calculate(evaluator, types, operator):
-    """
-    Calculates `+`, `-`, `~` and `not` prefixes.
-    """
-    for typ in types:
-        if operator == '-':
-            if _is_number(typ):
-                yield create(evaluator, -typ.obj)
-        elif operator == 'not':
-            value = typ.py__bool__()
-            if value is None:  # Uncertainty.
-                return
-            yield create(evaluator, not value)
-        else:
-            yield typ
-
-
-def _is_number(obj):
-    return isinstance(obj, CompiledObject) \
-        and isinstance(obj.obj, (int, float))
-
-
-def is_string(obj):
-    return isinstance(obj, CompiledObject) \
-        and isinstance(obj.obj, (str, unicode))
-
-
-def is_literal(obj):
-    return _is_number(obj) or is_string(obj)
-
-
-def _is_tuple(obj):
-    from jedi.evaluate import iterable
-    return isinstance(obj, iterable.AbstractSequence) and obj.array_type == 'tuple'
-
-
-def _is_list(obj):
-    from jedi.evaluate import iterable
-    return isinstance(obj, iterable.AbstractSequence) and obj.array_type == 'list'
-
-
-def _element_calculate(evaluator, context, left, operator, right):
-    from jedi.evaluate import iterable, instance
-    l_is_num = _is_number(left)
-    r_is_num = _is_number(right)
-    if operator == '*':
-        # for iterables, ignore * operations
-        if isinstance(left, iterable.AbstractSequence) or is_string(left):
-            return set([left])
-        elif isinstance(right, iterable.AbstractSequence) or is_string(right):
-            return set([right])
-    elif operator == '+':
-        if l_is_num and r_is_num or is_string(left) and is_string(right):
-            return set([create(evaluator, left.obj + right.obj)])
-        elif _is_tuple(left) and _is_tuple(right) or _is_list(left) and _is_list(right):
-            return set([iterable.MergedArray(evaluator, (left, right))])
-    elif operator == '-':
-        if l_is_num and r_is_num:
-            return set([create(evaluator, left.obj - right.obj)])
-    elif operator == '%':
-        # With strings and numbers the left type typically remains. Except for
-        # `int() % float()`.
-        return set([left])
-    elif operator in COMPARISON_OPERATORS:
-        operation = COMPARISON_OPERATORS[operator]
-        if isinstance(left, CompiledObject) and isinstance(right, CompiledObject):
-            # Possible, because the return is not an option. Just compare.
-            left = left.obj
-            right = right.obj
-
-        try:
-            result = operation(left, right)
-        except TypeError:
-            # Could be True or False.
-            return set([create(evaluator, True), create(evaluator, False)])
-        else:
-            return set([create(evaluator, result)])
-    elif operator == 'in':
-        return set()
-
-    def check(obj):
-        """Checks if a Jedi object is either a float or an int."""
-        return isinstance(obj, instance.CompiledInstance) and \
-            obj.name.string_name in ('int', 'float')
-
-    # Static analysis, one is a number, the other one is not.
-    if operator in ('+', '-') and l_is_num != r_is_num \
-            and not (check(left) or check(right)):
-        message = "TypeError: unsupported operand type(s) for +: %s and %s"
-        analysis.add(context, 'type-error-operation', operator,
-                     message % (left, right))
-
-    return set([left, right])

jedi/evaluate/recursion.py

@@ -29,6 +29,7 @@ therefore the quality might not always be maximal.
 from contextlib import contextmanager
 
 from jedi import debug
+from jedi.evaluate.base_context import NO_CONTEXTS
 
 
 recursion_limit = 15
@@ -48,6 +49,7 @@ per_function_recursion_limit = 2
 A function may not be executed more than this number of times recursively.
 """
 
+
 class RecursionDetector(object):
     def __init__(self):
         self.pushed_nodes = []
@@ -63,24 +65,26 @@ def execution_allowed(evaluator, node):
 
     if node in pushed_nodes:
         debug.warning('catched stmt recursion: %s @%s', node,
-                      node.start_pos)
+                      getattr(node, 'start_pos', None))
         yield False
     else:
-        pushed_nodes.append(node)
-        yield True
-        pushed_nodes.pop()
+        try:
+            pushed_nodes.append(node)
+            yield True
+        finally:
+            pushed_nodes.pop()
 
 
-def execution_recursion_decorator(default=set()):
+def execution_recursion_decorator(default=NO_CONTEXTS):
     def decorator(func):
-        def wrapper(execution, **kwargs):
-            detector = execution.evaluator.execution_recursion_detector
-            allowed = detector.push_execution(execution)
+        def wrapper(self, **kwargs):
+            detector = self.evaluator.execution_recursion_detector
+            allowed = detector.push_execution(self)
             try:
                 if allowed:
                     result = default
                 else:
-                    result = func(execution, **kwargs)
+                    result = func(self, **kwargs)
             finally:
                 detector.pop_execution()
             return result
@@ -112,7 +116,7 @@ class ExecutionRecursionDetector(object):
         self._parent_execution_funcs.append(funcdef)
 
         module = execution.get_root_context()
-        if module == self._evaluator.BUILTINS:
+        if module == self._evaluator.builtins_module:
             # We have control over builtins so we know they are not recursing
             # like crazy. Therefore we just let them execute always, because
             # they usually just help a lot with getting good results.

jedi/evaluate/representation.py

@@ -1,684 +0,0 @@
-"""
-Like described in the :mod:`parso.python.tree` module,
-there's a need for an ast like module to represent the states of parsed
-modules.
-
-But now there are also structures in Python that need a little bit more than
-that. An ``Instance`` for example is only a ``Class`` before it is
-instantiated. This class represents these cases.
-
-So, why is there also a ``Class`` class here? Well, there are decorators and
-they change classes in Python 3.
-
-Representation modules also define "magic methods". Those methods look like
-``py__foo__`` and are typically mappable to the Python equivalents ``__call__``
-and others. Here's a list:
-
-====================================== ========================================
-**Method**                             **Description**
--------------------------------------- ----------------------------------------
-py__call__(params: Array)              On callable objects, returns types.
-py__bool__()                           Returns True/False/None; None means that
-                                       there's no certainty.
-py__bases__()                          Returns a list of base classes.
-py__mro__()                            Returns a list of classes (the mro).
-py__iter__()                           Returns a generator of a set of types.
-py__class__()                          Returns the class of an instance.
-py__getitem__(index: int/str)          Returns a a set of types of the index.
-                                       Can raise an IndexError/KeyError.
-py__file__()                           Only on modules. Returns None if does
-                                       not exist.
-py__package__()                        Only on modules. For the import system.
-py__path__()                           Only on modules. For the import system.
-py__get__(call_object)                 Only on instances. Simulates
-                                       descriptors.
-py__doc__(include_call_signature:      Returns the docstring for a context.
-          bool)
-====================================== ========================================
-
-"""
-import os
-import pkgutil
-import imp
-import re
-from itertools import chain
-
-from parso.python import tree
-from parso import python_bytes_to_unicode
-
-from jedi._compatibility import use_metaclass
-from jedi import debug
-from jedi.evaluate.cache import evaluator_method_cache, CachedMetaClass
-from jedi.evaluate import compiled
-from jedi.evaluate import recursion
-from jedi.evaluate import iterable
-from jedi.evaluate import docstrings
-from jedi.evaluate import pep0484
-from jedi.evaluate import param
-from jedi.evaluate import flow_analysis
-from jedi.evaluate import imports
-from jedi.evaluate import helpers
-from jedi.evaluate.filters import ParserTreeFilter, FunctionExecutionFilter, \
-    GlobalNameFilter, DictFilter, ContextName, AbstractNameDefinition, \
-    ParamName, AnonymousInstanceParamName, TreeNameDefinition, \
-    ContextNameMixin
-from jedi.evaluate import context
-from jedi.evaluate.context import ContextualizedNode
-from jedi import parser_utils
-from jedi.evaluate.parser_cache import get_yield_exprs
-
-
-def apply_py__get__(context, base_context):
-    try:
-        method = context.py__get__
-    except AttributeError:
-        yield context
-    else:
-        for descriptor_context in method(base_context):
-            yield descriptor_context
-
-
-class ClassName(TreeNameDefinition):
-    def __init__(self, parent_context, tree_name, name_context):
-        super(ClassName, self).__init__(parent_context, tree_name)
-        self._name_context = name_context
-
-    def infer(self):
-        # TODO this _name_to_types might get refactored and be a part of the
-        # parent class. Once it is, we can probably just overwrite method to
-        # achieve this.
-        from jedi.evaluate.finder import _name_to_types
-        inferred = _name_to_types(
-            self.parent_context.evaluator, self._name_context, self.tree_name)
-
-        for result_context in inferred:
-            for c in apply_py__get__(result_context, self.parent_context):
-                yield c
-
-
-class ClassFilter(ParserTreeFilter):
-    name_class = ClassName
-
-    def _convert_names(self, names):
-        return [self.name_class(self.context, name, self._node_context)
-                for name in names]
-
-
-class ClassContext(use_metaclass(CachedMetaClass, context.TreeContext)):
-    """
-    This class is not only important to extend `tree.Class`, it is also a
-    important for descriptors (if the descriptor methods are evaluated or not).
-    """
-    api_type = 'class'
-
-    def __init__(self, evaluator, classdef, parent_context):
-        super(ClassContext, self).__init__(evaluator, parent_context=parent_context)
-        self.tree_node = classdef
-
-    @evaluator_method_cache(default=())
-    def py__mro__(self):
-        def add(cls):
-            if cls not in mro:
-                mro.append(cls)
-
-        mro = [self]
-        # TODO Do a proper mro resolution. Currently we are just listing
-        # classes. However, it's a complicated algorithm.
-        for lazy_cls in self.py__bases__():
-            # TODO there's multiple different mro paths possible if this yields
-            # multiple possibilities. Could be changed to be more correct.
-            for cls in lazy_cls.infer():
-                # TODO detect for TypeError: duplicate base class str,
-                # e.g.  `class X(str, str): pass`
-                try:
-                    mro_method = cls.py__mro__
-                except AttributeError:
-                    # TODO add a TypeError like:
-                    """
-                    >>> class Y(lambda: test): pass
-                    Traceback (most recent call last):
-                      File "<stdin>", line 1, in <module>
-                    TypeError: function() argument 1 must be code, not str
-                    >>> class Y(1): pass
-                    Traceback (most recent call last):
-                      File "<stdin>", line 1, in <module>
-                    TypeError: int() takes at most 2 arguments (3 given)
-                    """
-                    pass
-                else:
-                    add(cls)
-                    for cls_new in mro_method():
-                        add(cls_new)
-        return tuple(mro)
-
-    @evaluator_method_cache(default=())
-    def py__bases__(self):
-        arglist = self.tree_node.get_super_arglist()
-        if arglist:
-            args = param.TreeArguments(self.evaluator, self, arglist)
-            return [value for key, value in args.unpack() if key is None]
-        else:
-            return [context.LazyKnownContext(compiled.create(self.evaluator, object))]
-
-    def py__call__(self, params):
-        from jedi.evaluate.instance import TreeInstance
-        return set([TreeInstance(self.evaluator, self.parent_context, self, params)])
-
-    def py__class__(self):
-        return compiled.create(self.evaluator, type)
-
-    def get_params(self):
-        from jedi.evaluate.instance import AnonymousInstance
-        anon = AnonymousInstance(self.evaluator, self.parent_context, self)
-        return [AnonymousInstanceParamName(anon, param.name) for param in self.funcdef.get_params()]
-
-    def get_filters(self, search_global, until_position=None, origin_scope=None, is_instance=False):
-        if search_global:
-            yield ParserTreeFilter(
-                self.evaluator,
-                context=self,
-                until_position=until_position,
-                origin_scope=origin_scope
-            )
-        else:
-            for cls in self.py__mro__():
-                if isinstance(cls, compiled.CompiledObject):
-                    for filter in cls.get_filters(is_instance=is_instance):
-                        yield filter
-                else:
-                    yield ClassFilter(
-                        self.evaluator, self, node_context=cls,
-                        origin_scope=origin_scope)
-
-    def is_class(self):
-        return True
-
-    def get_function_slot_names(self, name):
-        for filter in self.get_filters(search_global=False):
-            names = filter.get(name)
-            if names:
-                return names
-        return []
-
-    def get_param_names(self):
-        for name in self.get_function_slot_names('__init__'):
-            for context_ in name.infer():
-                try:
-                    method = context_.get_param_names
-                except AttributeError:
-                    pass
-                else:
-                    return list(method())[1:]
-        return []
-
-    @property
-    def name(self):
-        return ContextName(self, self.tree_node.name)
-
-
-class LambdaName(AbstractNameDefinition):
-    string_name = '<lambda>'
-
-    def __init__(self, lambda_context):
-        self._lambda_context = lambda_context
-        self.parent_context = lambda_context.parent_context
-
-    def start_pos(self):
-        return self._lambda_context.tree_node.start_pos
-
-    def infer(self):
-        return set([self._lambda_context])
-
-
-class FunctionContext(use_metaclass(CachedMetaClass, context.TreeContext)):
-    """
-    Needed because of decorators. Decorators are evaluated here.
-    """
-    api_type = 'function'
-
-    def __init__(self, evaluator, parent_context, funcdef):
-        """ This should not be called directly """
-        super(FunctionContext, self).__init__(evaluator, parent_context)
-        self.tree_node = funcdef
-
-    def get_filters(self, search_global, until_position=None, origin_scope=None):
-        if search_global:
-            yield ParserTreeFilter(
-                self.evaluator,
-                context=self,
-                until_position=until_position,
-                origin_scope=origin_scope
-            )
-        else:
-            scope = self.py__class__()
-            for filter in scope.get_filters(search_global=False, origin_scope=origin_scope):
-                yield filter
-
-    def infer_function_execution(self, function_execution):
-        """
-        Created to be used by inheritance.
-        """
-        yield_exprs = get_yield_exprs(self.evaluator, self.tree_node)
-        if yield_exprs:
-            return set([iterable.Generator(self.evaluator, function_execution)])
-        else:
-            return function_execution.get_return_values()
-
-    def get_function_execution(self, arguments=None):
-        if arguments is None:
-            arguments = param.AnonymousArguments()
-
-        return FunctionExecutionContext(self.evaluator, self.parent_context, self, arguments)
-
-    def py__call__(self, arguments):
-        function_execution = self.get_function_execution(arguments)
-        return self.infer_function_execution(function_execution)
-
-    def py__class__(self):
-        # This differentiation is only necessary for Python2. Python3 does not
-        # use a different method class.
-        if isinstance(parser_utils.get_parent_scope(self.tree_node), tree.Class):
-            name = 'METHOD_CLASS'
-        else:
-            name = 'FUNCTION_CLASS'
-        return compiled.get_special_object(self.evaluator, name)
-
-    @property
-    def name(self):
-        if self.tree_node.type == 'lambdef':
-            return LambdaName(self)
-        return ContextName(self, self.tree_node.name)
-
-    def get_param_names(self):
-        function_execution = self.get_function_execution()
-        return [ParamName(function_execution, param.name)
-                for param in self.tree_node.get_params()]
-
-
-class FunctionExecutionContext(context.TreeContext):
-    """
-    This class is used to evaluate functions and their returns.
-
-    This is the most complicated class, because it contains the logic to
-    transfer parameters. It is even more complicated, because there may be
-    multiple calls to functions and recursion has to be avoided. But this is
-    responsibility of the decorators.
-    """
-    function_execution_filter = FunctionExecutionFilter
-
-    def __init__(self, evaluator, parent_context, function_context, var_args):
-        super(FunctionExecutionContext, self).__init__(evaluator, parent_context)
-        self.function_context = function_context
-        self.tree_node = function_context.tree_node
-        self.var_args = var_args
-
-    @evaluator_method_cache(default=set())
-    @recursion.execution_recursion_decorator()
-    def get_return_values(self, check_yields=False):
-        funcdef = self.tree_node
-        if funcdef.type == 'lambdef':
-            return self.evaluator.eval_element(self, funcdef.children[-1])
-
-        if check_yields:
-            types = set()
-            returns = get_yield_exprs(self.evaluator, funcdef)
-        else:
-            returns = funcdef.iter_return_stmts()
-            types = set(docstrings.infer_return_types(self.function_context))
-            types |= set(pep0484.infer_return_types(self.function_context))
-
-        for r in returns:
-            check = flow_analysis.reachability_check(self, funcdef, r)
-            if check is flow_analysis.UNREACHABLE:
-                debug.dbg('Return unreachable: %s', r)
-            else:
-                if check_yields:
-                    types |= set(self._eval_yield(r))
-                else:
-                    try:
-                        children = r.children
-                    except AttributeError:
-                        types.add(compiled.create(self.evaluator, None))
-                    else:
-                        types |= self.eval_node(children[1])
-            if check is flow_analysis.REACHABLE:
-                debug.dbg('Return reachable: %s', r)
-                break
-        return types
-
-    def _eval_yield(self, yield_expr):
-        if yield_expr.type == 'keyword':
-            # `yield` just yields None.
-            yield context.LazyKnownContext(compiled.create(self.evaluator, None))
-            return
-
-        node = yield_expr.children[1]
-        if node.type == 'yield_arg':  # It must be a yield from.
-            cn = ContextualizedNode(self, node.children[1])
-            for lazy_context in iterable.py__iter__(self.evaluator, cn.infer(), cn):
-                yield lazy_context
-        else:
-            yield context.LazyTreeContext(self, node)
-
-    @recursion.execution_recursion_decorator(default=iter([]))
-    def get_yield_values(self):
-        for_parents = [(y, tree.search_ancestor(y, 'for_stmt', 'funcdef',
-                                                'while_stmt', 'if_stmt'))
-                       for y in get_yield_exprs(self.evaluator, self.tree_node)]
-
-        # Calculate if the yields are placed within the same for loop.
-        yields_order = []
-        last_for_stmt = None
-        for yield_, for_stmt in for_parents:
-            # For really simple for loops we can predict the order. Otherwise
-            # we just ignore it.
-            parent = for_stmt.parent
-            if parent.type == 'suite':
-                parent = parent.parent
-            if for_stmt.type == 'for_stmt' and parent == self.tree_node \
-                    and parser_utils.for_stmt_defines_one_name(for_stmt):  # Simplicity for now.
-                if for_stmt == last_for_stmt:
-                    yields_order[-1][1].append(yield_)
-                else:
-                    yields_order.append((for_stmt, [yield_]))
-            elif for_stmt == self.tree_node:
-                yields_order.append((None, [yield_]))
-            else:
-                types = self.get_return_values(check_yields=True)
-                if types:
-                    yield context.get_merged_lazy_context(list(types))
-                return
-            last_for_stmt = for_stmt
-
-        evaluator = self.evaluator
-        for for_stmt, yields in yields_order:
-            if for_stmt is None:
-                # No for_stmt, just normal yields.
-                for yield_ in yields:
-                    for result in self._eval_yield(yield_):
-                        yield result
-            else:
-                input_node = for_stmt.get_testlist()
-                cn = ContextualizedNode(self, input_node)
-                ordered = iterable.py__iter__(evaluator, cn.infer(), cn)
-                ordered = list(ordered)
-                for lazy_context in ordered:
-                    dct = {str(for_stmt.children[1].value): lazy_context.infer()}
-                    with helpers.predefine_names(self, for_stmt, dct):
-                        for yield_in_same_for_stmt in yields:
-                            for result in self._eval_yield(yield_in_same_for_stmt):
-                                yield result
-
-    def get_filters(self, search_global, until_position=None, origin_scope=None):
-        yield self.function_execution_filter(self.evaluator, self,
-                                             until_position=until_position,
-                                             origin_scope=origin_scope)
-
-    @evaluator_method_cache()
-    def get_params(self):
-        return self.var_args.get_params(self)
-
-
-class ModuleAttributeName(AbstractNameDefinition):
-    """
-    For module attributes like __file__, __str__ and so on.
-    """
-    api_type = 'instance'
-
-    def __init__(self, parent_module, string_name):
-        self.parent_context = parent_module
-        self.string_name = string_name
-
-    def infer(self):
-        return compiled.create(self.parent_context.evaluator, str).execute(
-            param.ValuesArguments([])
-        )
-
-
-class ModuleName(ContextNameMixin, AbstractNameDefinition):
-    start_pos = 1, 0
-
-    def __init__(self, context, name):
-        self._context = context
-        self._name = name
-
-    @property
-    def string_name(self):
-        return self._name
-
-
-class ModuleContext(use_metaclass(CachedMetaClass, context.TreeContext)):
-    api_type = 'module'
-    parent_context = None
-
-    def __init__(self, evaluator, module_node, path):
-        super(ModuleContext, self).__init__(evaluator, parent_context=None)
-        self.tree_node = module_node
-        self._path = path
-
-    def get_filters(self, search_global, until_position=None, origin_scope=None):
-        yield ParserTreeFilter(
-            self.evaluator,
-            context=self,
-            until_position=until_position,
-            origin_scope=origin_scope
-        )
-        yield GlobalNameFilter(self, self.tree_node)
-        yield DictFilter(self._sub_modules_dict())
-        yield DictFilter(self._module_attributes_dict())
-        for star_module in self.star_imports():
-            yield next(star_module.get_filters(search_global))
-
-    # I'm not sure if the star import cache is really that effective anymore
-    # with all the other really fast import caches. Recheck. Also we would need
-    # to push the star imports into Evaluator.modules, if we reenable this.
-    @evaluator_method_cache([])
-    def star_imports(self):
-        modules = []
-        for i in self.tree_node.iter_imports():
-            if i.is_star_import():
-                name = i.get_paths()[-1][-1]
-                new = imports.infer_import(self, name)
-                for module in new:
-                    if isinstance(module, ModuleContext):
-                        modules += module.star_imports()
-                modules += new
-        return modules
-
-    @evaluator_method_cache()
-    def _module_attributes_dict(self):
-        names = ['__file__', '__package__', '__doc__', '__name__']
-        # All the additional module attributes are strings.
-        return dict((n, ModuleAttributeName(self, n)) for n in names)
-
-    @property
-    def _string_name(self):
-        """ This is used for the goto functions. """
-        if self._path is None:
-            return ''  # no path -> empty name
-        else:
-            sep = (re.escape(os.path.sep),) * 2
-            r = re.search(r'([^%s]*?)(%s__init__)?(\.py|\.so)?$' % sep, self._path)
-            # Remove PEP 3149 names
-            return re.sub('\.[a-z]+-\d{2}[mud]{0,3}$', '', r.group(1))
-
-    @property
-    @evaluator_method_cache()
-    def name(self):
-        return ModuleName(self, self._string_name)
-
-    def _get_init_directory(self):
-        """
-        :return: The path to the directory of a package. None in case it's not
-                 a package.
-        """
-        for suffix, _, _ in imp.get_suffixes():
-            ending = '__init__' + suffix
-            py__file__ = self.py__file__()
-            if py__file__ is not None and py__file__.endswith(ending):
-                # Remove the ending, including the separator.
-                return self.py__file__()[:-len(ending) - 1]
-        return None
-
-    def py__name__(self):
-        for name, module in self.evaluator.modules.items():
-            if module == self and name != '':
-                return name
-
-        return '__main__'
-
-    def py__file__(self):
-        """
-        In contrast to Python's __file__ can be None.
-        """
-        if self._path is None:
-            return None
-
-        return os.path.abspath(self._path)
-
-    def py__package__(self):
-        if self._get_init_directory() is None:
-            return re.sub(r'\.?[^\.]+$', '', self.py__name__())
-        else:
-            return self.py__name__()
-
-    def _py__path__(self):
-        search_path = self.evaluator.sys_path
-        init_path = self.py__file__()
-        if os.path.basename(init_path) == '__init__.py':
-            with open(init_path, 'rb') as f:
-                content = python_bytes_to_unicode(f.read(), errors='replace')
-                # these are strings that need to be used for namespace packages,
-                # the first one is ``pkgutil``, the second ``pkg_resources``.
-                options = ('declare_namespace(__name__)', 'extend_path(__path__')
-                if options[0] in content or options[1] in content:
-                    # It is a namespace, now try to find the rest of the
-                    # modules on sys_path or whatever the search_path is.
-                    paths = set()
-                    for s in search_path:
-                        other = os.path.join(s, self.name.string_name)
-                        if os.path.isdir(other):
-                            paths.add(other)
-                    if paths:
-                        return list(paths)
-                    # TODO I'm not sure if this is how nested namespace
-                    # packages work. The tests are not really good enough to
-                    # show that.
-        # Default to this.
-        return [self._get_init_directory()]
-
-    @property
-    def py__path__(self):
-        """
-        Not seen here, since it's a property. The callback actually uses a
-        variable, so use it like::
-
-            foo.py__path__(sys_path)
-
-        In case of a package, this returns Python's __path__ attribute, which
-        is a list of paths (strings).
-        Raises an AttributeError if the module is not a package.
-        """
-        path = self._get_init_directory()
-
-        if path is None:
-            raise AttributeError('Only packages have __path__ attributes.')
-        else:
-            return self._py__path__
-
-    @evaluator_method_cache()
-    def _sub_modules_dict(self):
-        """
-        Lists modules in the directory of this module (if this module is a
-        package).
-        """
-        path = self._path
-        names = {}
-        if path is not None and path.endswith(os.path.sep + '__init__.py'):
-            mods = pkgutil.iter_modules([os.path.dirname(path)])
-            for module_loader, name, is_pkg in mods:
-                # It's obviously a relative import to the current module.
-                names[name] = imports.SubModuleName(self, name)
-
-        # TODO add something like this in the future, its cleaner than the
-        #   import hacks.
-        # ``os.path`` is a hardcoded exception, because it's a
-        # ``sys.modules`` modification.
-        # if str(self.name) == 'os':
-        #     names.append(Name('path', parent_context=self))
-
-        return names
-
-    def py__class__(self):
-        return compiled.get_special_object(self.evaluator, 'MODULE_CLASS')
-
-    def __repr__(self):
-        return "<%s: %s@%s-%s>" % (
-            self.__class__.__name__, self._string_name,
-            self.tree_node.start_pos[0], self.tree_node.end_pos[0])
-
-
-class ImplicitNSName(AbstractNameDefinition):
-    """
-    Accessing names for implicit namespace packages should infer to nothing.
-    This object will prevent Jedi from raising exceptions
-    """
-    def __init__(self, implicit_ns_context, string_name):
-        self.implicit_ns_context = implicit_ns_context
-        self.string_name = string_name
-
-    def infer(self):
-        return []
-
-    def get_root_context(self):
-        return self.implicit_ns_context
-
-
-class ImplicitNamespaceContext(use_metaclass(CachedMetaClass, context.TreeContext)):
-    """
-    Provides support for implicit namespace packages
-    """
-    api_type = 'module'
-    parent_context = None
-
-    def __init__(self, evaluator, fullname):
-        super(ImplicitNamespaceContext, self).__init__(evaluator, parent_context=None)
-        self.evaluator = evaluator
-        self.fullname = fullname
-
-    def get_filters(self, search_global, until_position=None, origin_scope=None):
-        yield DictFilter(self._sub_modules_dict())
-
-    @property
-    @evaluator_method_cache()
-    def name(self):
-        string_name = self.py__package__().rpartition('.')[-1]
-        return ImplicitNSName(self, string_name)
-
-    def py__file__(self):
-        return None
-
-    def py__package__(self):
-        """Return the fullname
-        """
-        return self.fullname
-
-    @property
-    def py__path__(self):
-        return lambda: [self.paths]
-
-    @evaluator_method_cache()
-    def _sub_modules_dict(self):
-        names = {}
-
-        paths = self.paths
-        file_names = chain.from_iterable(os.listdir(path) for path in paths)
-        mods = [
-            file_name.rpartition('.')[0] if '.' in file_name else file_name
-            for file_name in file_names
-            if file_name != '__pycache__'
-        ]
-
-        for name in mods:
-            names[name] = imports.SubModuleName(self, name)
-        return names

jedi/evaluate/site.py

@@ -1,110 +0,0 @@
-"""An adapted copy of relevant site-packages functionality from Python stdlib.
-
-This file contains some functions related to handling site-packages in Python
-with jedi-specific modifications:
-
-- the functions operate on sys_path argument rather than global sys.path
-
-- in .pth files "import ..." lines that allow execution of arbitrary code are
-  skipped to prevent code injection into jedi interpreter
-
-"""
-
-# Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010,
-# 2011, 2012, 2013, 2014, 2015 Python Software Foundation; All Rights Reserved
-
-from __future__ import print_function
-
-import sys
-import os
-
-
-def makepath(*paths):
-    dir = os.path.join(*paths)
-    try:
-        dir = os.path.abspath(dir)
-    except OSError:
-        pass
-    return dir, os.path.normcase(dir)
-
-
-def _init_pathinfo(sys_path):
-    """Return a set containing all existing directory entries from sys_path"""
-    d = set()
-    for dir in sys_path:
-        try:
-            if os.path.isdir(dir):
-                dir, dircase = makepath(dir)
-                d.add(dircase)
-        except TypeError:
-            continue
-    return d
-
-
-def addpackage(sys_path, sitedir, name, known_paths):
-    """Process a .pth file within the site-packages directory:
-       For each line in the file, either combine it with sitedir to a path
-       and add that to known_paths, or execute it if it starts with 'import '.
-    """
-    if known_paths is None:
-        known_paths = _init_pathinfo(sys_path)
-        reset = 1
-    else:
-        reset = 0
-    fullname = os.path.join(sitedir, name)
-    try:
-        f = open(fullname, "r")
-    except OSError:
-        return
-    with f:
-        for n, line in enumerate(f):
-            if line.startswith("#"):
-                continue
-            try:
-                if line.startswith(("import ", "import\t")):
-                    # Change by immerrr: don't evaluate import lines to prevent
-                    # code injection into jedi through pth files.
-                    #
-                    # exec(line)
-                    continue
-                line = line.rstrip()
-                dir, dircase = makepath(sitedir, line)
-                if not dircase in known_paths and os.path.exists(dir):
-                    sys_path.append(dir)
-                    known_paths.add(dircase)
-            except Exception:
-                print("Error processing line {:d} of {}:\n".format(n+1, fullname),
-                      file=sys.stderr)
-                import traceback
-                for record in traceback.format_exception(*sys.exc_info()):
-                    for line in record.splitlines():
-                        print('  '+line, file=sys.stderr)
-                print("\nRemainder of file ignored", file=sys.stderr)
-                break
-    if reset:
-        known_paths = None
-    return known_paths
-
-
-def addsitedir(sys_path, sitedir, known_paths=None):
-    """Add 'sitedir' argument to sys_path if missing and handle .pth files in
-    'sitedir'"""
-    if known_paths is None:
-        known_paths = _init_pathinfo(sys_path)
-        reset = 1
-    else:
-        reset = 0
-    sitedir, sitedircase = makepath(sitedir)
-    if not sitedircase in known_paths:
-        sys_path.append(sitedir)        # Add path component
-        known_paths.add(sitedircase)
-    try:
-        names = os.listdir(sitedir)
-    except OSError:
-        return
-    names = [name for name in names if name.endswith(".pth")]
-    for name in sorted(names):
-        addpackage(sys_path, sitedir, name, known_paths)
-    if reset:
-        known_paths = None
-    return known_paths

jedi/evaluate/stdlib.py

@@ -9,21 +9,21 @@ Note that this module exists only to implement very specific functionality in
 the standard library. The usual way to understand the standard library is the
 compiled module that returns the types for C-builtins.
 """
-import collections
-import re
+import parso
 
-from jedi.common import unite
-from jedi.evaluate import compiled
-from jedi.evaluate import representation as er
-from jedi.evaluate.instance import InstanceFunctionExecution, \
-    AbstractInstanceContext, CompiledInstance, BoundMethod, \
-    AnonymousInstanceFunctionExecution
-from jedi.evaluate import iterable
+from jedi._compatibility import force_unicode
 from jedi import debug
-from jedi.evaluate import precedence
-from jedi.evaluate import param
+from jedi.evaluate.arguments import ValuesArguments, repack_with_argument_clinic
 from jedi.evaluate import analysis
-from jedi.evaluate.context import LazyTreeContext, ContextualizedNode
+from jedi.evaluate import compiled
+from jedi.evaluate.context.instance import \
+    AbstractInstanceContext, CompiledInstance, BoundMethod, InstanceArguments
+from jedi.evaluate.base_context import ContextualizedNode, \
+    NO_CONTEXTS, ContextSet
+from jedi.evaluate.context import ClassContext, ModuleContext, FunctionExecutionContext
+from jedi.evaluate.context import iterable
+from jedi.evaluate.lazy_context import LazyTreeContext
+from jedi.evaluate.syntax_tree import is_string
 
 # Now this is all part of fake tuples in Jedi. However super doesn't work on
 # __init__ and __new__ doesn't work at all. So adding this to nametuples is
@@ -56,9 +56,9 @@ def execute(evaluator, obj, arguments):
     except AttributeError:
         pass
     else:
-        if obj.parent_context == evaluator.BUILTINS:
+        if obj.parent_context == evaluator.builtins_module:
             module_name = 'builtins'
-        elif isinstance(obj.parent_context, er.ModuleContext):
+        elif isinstance(obj.parent_context, ModuleContext):
             module_name = obj.parent_context.name.string_name
         else:
             module_name = ''
@@ -69,7 +69,7 @@ def execute(evaluator, obj, arguments):
         except KeyError:
             pass
         else:
-            return func(evaluator, obj, arguments)
+            return func(evaluator, obj, arguments=arguments)
     raise NotInStdLib()
 
 
@@ -77,7 +77,7 @@ def _follow_param(evaluator, arguments, index):
     try:
         key, lazy_context = list(arguments.unpack())[index]
     except IndexError:
-        return set()
+        return NO_CONTEXTS
     else:
         return lazy_context.infer()
 
@@ -86,41 +86,23 @@ def argument_clinic(string, want_obj=False, want_context=False, want_arguments=F
     """
     Works like Argument Clinic (PEP 436), to validate function params.
     """
-    clinic_args = []
-    allow_kwargs = False
-    optional = False
-    while string:
-        # Optional arguments have to begin with a bracket. And should always be
-        # at the end of the arguments. This is therefore not a proper argument
-        # clinic implementation. `range()` for exmple allows an optional start
-        # value at the beginning.
-        match = re.match('(?:(?:(\[),? ?|, ?|)(\w+)|, ?/)\]*', string)
-        string = string[len(match.group(0)):]
-        if not match.group(2):  # A slash -> allow named arguments
-            allow_kwargs = True
-            continue
-        optional = optional or bool(match.group(1))
-        word = match.group(2)
-        clinic_args.append((word, optional, allow_kwargs))
 
     def f(func):
-        def wrapper(evaluator, obj, arguments):
+        @repack_with_argument_clinic(string, keep_arguments_param=True)
+        def wrapper(evaluator, obj, *args, **kwargs):
+            arguments = kwargs.pop('arguments')
+            assert not kwargs  # Python 2...
             debug.dbg('builtin start %s' % obj, color='MAGENTA')
-            try:
-                lst = list(arguments.eval_argument_clinic(clinic_args))
-            except ValueError:
-                return set()
-            else:
-                kwargs = {}
-                if want_context:
-                    kwargs['context'] = arguments.context
-                if want_obj:
-                    kwargs['obj'] = obj
-                if want_arguments:
-                    kwargs['arguments'] = arguments
-                return func(evaluator, *lst, **kwargs)
-            finally:
-                debug.dbg('builtin end', color='MAGENTA')
+            result = NO_CONTEXTS
+            if want_context:
+                kwargs['context'] = arguments.context
+            if want_obj:
+                kwargs['obj'] = obj
+            if want_arguments:
+                kwargs['arguments'] = arguments
+            result = func(evaluator, *args, **kwargs)
+            debug.dbg('builtin end: %s', result, color='MAGENTA')
+            return result
 
         return wrapper
     return f
@@ -132,20 +114,21 @@ def builtins_next(evaluator, iterators, defaults):
     TODO this function is currently not used. It's a stab at implementing next
     in a different way than fake objects. This would be a bit more flexible.
     """
-    if evaluator.python_version[0] == 2:
+    if evaluator.environment.version_info.major == 2:
         name = 'next'
     else:
         name = '__next__'
 
-    types = set()
+    context_set = NO_CONTEXTS
     for iterator in iterators:
         if isinstance(iterator, AbstractInstanceContext):
-            for filter in iterator.get_filters(include_self_names=True):
-                for n in filter.get(name):
-                    for context in n.infer():
-                        types |= context.execute_evaluated()
-    if types:
-        return types
+            context_set = ContextSet.from_sets(
+                n.infer()
+                for filter in iterator.get_filters(include_self_names=True)
+                for n in filter.get(name)
+            ).execute_evaluated()
+    if context_set:
+        return context_set
     return defaults
 
 
@@ -154,21 +137,21 @@ def builtins_getattr(evaluator, objects, names, defaults=None):
     # follow the first param
     for obj in objects:
         for name in names:
-            if precedence.is_string(name):
-                return obj.py__getattribute__(name.obj)
+            if is_string(name):
+                return obj.py__getattribute__(force_unicode(name.get_safe_value()))
             else:
                 debug.warning('getattr called without str')
                 continue
-    return set()
+    return NO_CONTEXTS
 
 
 @argument_clinic('object[, bases, dict], /')
 def builtins_type(evaluator, objects, bases, dicts):
     if bases or dicts:
         # It's a type creation... maybe someday...
-        return set()
+        return NO_CONTEXTS
     else:
-        return set([o.py__class__() for o in objects])
+        return objects.py__class__()
 
 
 class SuperInstance(AbstractInstanceContext):
@@ -181,11 +164,12 @@ class SuperInstance(AbstractInstanceContext):
 @argument_clinic('[type[, obj]], /', want_context=True)
 def builtins_super(evaluator, types, objects, context):
     # TODO make this able to detect multiple inheritance super
-    if isinstance(context, (InstanceFunctionExecution,
-                            AnonymousInstanceFunctionExecution)):
-        su = context.instance.py__class__().py__bases__()
-        return unite(context.execute_evaluated() for context in su[0].infer())
-    return set()
+    if isinstance(context, FunctionExecutionContext):
+        if isinstance(context.var_args, InstanceArguments):
+            su = context.var_args.instance.py__class__().py__bases__()
+            return su[0].infer().execute_evaluated()
+
+    return NO_CONTEXTS
 
 
 @argument_clinic('sequence, /', want_obj=True, want_arguments=True)
@@ -198,29 +182,31 @@ def builtins_reversed(evaluator, sequences, obj, arguments):
     if isinstance(lazy_context, LazyTreeContext):
         # TODO access private
         cn = ContextualizedNode(lazy_context._context, lazy_context.data)
-    ordered = list(iterable.py__iter__(evaluator, sequences, cn))
+    ordered = list(sequences.iterate(cn))
 
     rev = list(reversed(ordered))
     # Repack iterator values and then run it the normal way. This is
     # necessary, because `reversed` is a function and autocompletion
     # would fail in certain cases like `reversed(x).__iter__` if we
     # just returned the result directly.
-    seq = iterable.FakeSequence(evaluator, 'list', rev)
-    arguments = param.ValuesArguments([[seq]])
-    return set([CompiledInstance(evaluator, evaluator.BUILTINS, obj, arguments)])
+    seq = iterable.FakeSequence(evaluator, u'list', rev)
+    arguments = ValuesArguments([ContextSet(seq)])
+    return ContextSet(CompiledInstance(evaluator, evaluator.builtins_module, obj, arguments))
 
 
 @argument_clinic('obj, type, /', want_arguments=True)
 def builtins_isinstance(evaluator, objects, types, arguments):
-    bool_results = set([])
+    bool_results = set()
     for o in objects:
+        cls = o.py__class__()
         try:
-            mro_func = o.py__class__().py__mro__
+            mro_func = cls.py__mro__
         except AttributeError:
             # This is temporary. Everything should have a class attribute in
             # Python?! Maybe we'll leave it here, because some numpy objects or
             # whatever might not.
-            return set([compiled.create(True), compiled.create(False)])
+            bool_results = set([True, False])
+            break
 
         mro = mro_func()
 
@@ -228,11 +214,11 @@ def builtins_isinstance(evaluator, objects, types, arguments):
             if cls_or_tup.is_class():
                 bool_results.add(cls_or_tup in mro)
             elif cls_or_tup.name.string_name == 'tuple' \
-                    and cls_or_tup.get_root_context() == evaluator.BUILTINS:
+                    and cls_or_tup.get_root_context() == evaluator.builtins_module:
                 # Check for tuples.
-                classes = unite(
+                classes = ContextSet.from_sets(
                     lazy_context.infer()
-                    for lazy_context in cls_or_tup.py__iter__()
+                    for lazy_context in cls_or_tup.iterate()
                 )
                 bool_results.add(any(cls in mro for cls in classes))
             else:
@@ -244,7 +230,10 @@ def builtins_isinstance(evaluator, objects, types, arguments):
                               'not %s.' % cls_or_tup
                     analysis.add(lazy_context._context, 'type-error-isinstance', node, message)
 
-    return set(compiled.create(evaluator, x) for x in bool_results)
+    return ContextSet.from_iterable(
+        compiled.builtin_from_name(evaluator, force_unicode(str(b)))
+        for b in bool_results
+    )
 
 
 def collections_namedtuple(evaluator, obj, arguments):
@@ -254,46 +243,55 @@ def collections_namedtuple(evaluator, obj, arguments):
     This has to be done by processing the namedtuple class template and
     evaluating the result.
 
-    .. note:: |jedi| only supports namedtuples on Python >2.6.
-
     """
-    # Namedtuples are not supported on Python 2.6
-    if not hasattr(collections, '_class_template'):
-        return set()
+    collections_context = obj.parent_context
+    _class_template_set = collections_context.py__getattribute__(u'_class_template')
+    if not _class_template_set:
+        # Namedtuples are not supported on Python 2.6, early 2.7, because the
+        # _class_template variable is not defined, there.
+        return NO_CONTEXTS
 
     # Process arguments
     # TODO here we only use one of the types, we should use all.
-    name = list(_follow_param(evaluator, arguments, 0))[0].obj
+    # TODO this is buggy, doesn't need to be a string
+    name = list(_follow_param(evaluator, arguments, 0))[0].get_safe_value()
     _fields = list(_follow_param(evaluator, arguments, 1))[0]
     if isinstance(_fields, compiled.CompiledObject):
-        fields = _fields.obj.replace(',', ' ').split()
-    elif isinstance(_fields, iterable.AbstractSequence):
+        fields = _fields.get_safe_value().replace(',', ' ').split()
+    elif isinstance(_fields, iterable.Sequence):
         fields = [
-            v.obj
+            v.get_safe_value()
             for lazy_context in _fields.py__iter__()
-            for v in lazy_context.infer() if hasattr(v, 'obj')
+            for v in lazy_context.infer() if is_string(v)
         ]
     else:
-        return set()
+        return NO_CONTEXTS
 
-    base = collections._class_template
+    def get_var(name):
+        x, = collections_context.py__getattribute__(name)
+        return x.get_safe_value()
+
+    base = next(iter(_class_template_set)).get_safe_value()
     base += _NAMEDTUPLE_INIT
-    # Build source
-    source = base.format(
+    # Build source code
+    code = base.format(
         typename=name,
         field_names=tuple(fields),
         num_fields=len(fields),
-        arg_list = repr(tuple(fields)).replace("'", "")[1:-1],
-        repr_fmt=', '.join(collections._repr_template.format(name=name) for name in fields),
-        field_defs='\n'.join(collections._field_template.format(index=index, name=name)
+        arg_list=repr(tuple(fields)).replace("u'", "").replace("'", "")[1:-1],
+        repr_fmt=', '.join(get_var(u'_repr_template').format(name=name) for name in fields),
+        field_defs='\n'.join(get_var(u'_field_template').format(index=index, name=name)
                              for index, name in enumerate(fields))
     )
 
-    # Parse source
-    module = evaluator.grammar.parse(source)
+    # Parse source code
+    module = evaluator.grammar.parse(code)
     generated_class = next(module.iter_classdefs())
-    parent_context = er.ModuleContext(evaluator, module, '')
-    return set([er.ClassContext(evaluator, generated_class, parent_context)])
+    parent_context = ModuleContext(
+        evaluator, module, None,
+        code_lines=parso.split_lines(code, keepends=True),
+    )
+    return ContextSet(ClassContext(evaluator, parent_context, generated_class))
 
 
 @argument_clinic('first, /')
@@ -314,8 +312,8 @@ _implemented = {
         'deepcopy': _return_first_param,
     },
     'json': {
-        'load': lambda *args: set(),
-        'loads': lambda *args: set(),
+        'load': lambda evaluator, obj, arguments: NO_CONTEXTS,
+        'loads': lambda evaluator, obj, arguments: NO_CONTEXTS,
     },
     'collections': {
         'namedtuple': collections_namedtuple,

jedi/evaluate/syntax_tree.py

@@ -0,0 +1,662 @@
+"""
+Functions evaluating the syntax tree.
+"""
+import copy
+
+from parso.python import tree
+
+from jedi._compatibility import force_unicode, unicode
+from jedi import debug
+from jedi import parser_utils
+from jedi.evaluate.base_context import ContextSet, NO_CONTEXTS, ContextualizedNode, \
+    ContextualizedName, iterator_to_context_set, iterate_contexts
+from jedi.evaluate import compiled
+from jedi.evaluate import pep0484
+from jedi.evaluate import recursion
+from jedi.evaluate import helpers
+from jedi.evaluate import analysis
+from jedi.evaluate import imports
+from jedi.evaluate import arguments
+from jedi.evaluate.pep0484 import _evaluate_for_annotation
+from jedi.evaluate.context import ClassContext, FunctionContext
+from jedi.evaluate.context import iterable
+from jedi.evaluate.context import TreeInstance, CompiledInstance
+from jedi.evaluate.finder import NameFinder
+from jedi.evaluate.helpers import is_string, is_literal, is_number, is_compiled
+from jedi.evaluate.compiled.access import COMPARISON_OPERATORS
+
+
+def _limit_context_infers(func):
+    """
+    This is for now the way how we limit type inference going wild. There are
+    other ways to ensure recursion limits as well. This is mostly necessary
+    because of instance (self) access that can be quite tricky to limit.
+
+    I'm still not sure this is the way to go, but it looks okay for now and we
+    can still go anther way in the future. Tests are there. ~ dave
+    """
+    def wrapper(context, *args, **kwargs):
+        n = context.tree_node
+        evaluator = context.evaluator
+        try:
+            evaluator.inferred_element_counts[n] += 1
+            if evaluator.inferred_element_counts[n] > 300:
+                debug.warning('In context %s there were too many inferences.', n)
+                return NO_CONTEXTS
+        except KeyError:
+            evaluator.inferred_element_counts[n] = 1
+        return func(context, *args, **kwargs)
+
+    return wrapper
+
+
+def _py__stop_iteration_returns(generators):
+    results = ContextSet()
+    for generator in generators:
+        try:
+            method = generator.py__stop_iteration_returns
+        except AttributeError:
+            debug.warning('%s is not actually a generator', generator)
+        else:
+            results |= method()
+    return results
+
+
+@debug.increase_indent
+@_limit_context_infers
+def eval_node(context, element):
+    debug.dbg('eval_node %s@%s', element, element.start_pos)
+    evaluator = context.evaluator
+    typ = element.type
+    if typ in ('name', 'number', 'string', 'atom', 'strings', 'keyword'):
+        return eval_atom(context, element)
+    elif typ == 'lambdef':
+        return ContextSet(FunctionContext.from_context(context, element))
+    elif typ == 'expr_stmt':
+        return eval_expr_stmt(context, element)
+    elif typ in ('power', 'atom_expr'):
+        first_child = element.children[0]
+        children = element.children[1:]
+        had_await = False
+        if first_child.type == 'keyword' and first_child.value == 'await':
+            had_await = True
+            first_child = children.pop(0)
+
+        context_set = eval_atom(context, first_child)
+        for trailer in children:
+            if trailer == '**':  # has a power operation.
+                right = context.eval_node(children[1])
+                context_set = _eval_comparison(
+                    evaluator,
+                    context,
+                    context_set,
+                    trailer,
+                    right
+                )
+                break
+            context_set = eval_trailer(context, context_set, trailer)
+
+        if had_await:
+            await_context_set = context_set.py__getattribute__(u"__await__")
+            if not await_context_set:
+                debug.warning('Tried to run py__await__ on context %s', context)
+            context_set = ContextSet()
+            return _py__stop_iteration_returns(await_context_set.execute_evaluated())
+        return context_set
+    elif typ in ('testlist_star_expr', 'testlist',):
+        # The implicit tuple in statements.
+        return ContextSet(iterable.SequenceLiteralContext(evaluator, context, element))
+    elif typ in ('not_test', 'factor'):
+        context_set = context.eval_node(element.children[-1])
+        for operator in element.children[:-1]:
+            context_set = eval_factor(context_set, operator)
+        return context_set
+    elif typ == 'test':
+        # `x if foo else y` case.
+        return (context.eval_node(element.children[0]) |
+                context.eval_node(element.children[-1]))
+    elif typ == 'operator':
+        # Must be an ellipsis, other operators are not evaluated.
+        # In Python 2 ellipsis is coded as three single dot tokens, not
+        # as one token 3 dot token.
+        if element.value not in ('.', '...'):
+            origin = element.parent
+            raise AssertionError("unhandled operator %s in %s " % (repr(element.value), origin))
+        return ContextSet(compiled.builtin_from_name(evaluator, u'Ellipsis'))
+    elif typ == 'dotted_name':
+        context_set = eval_atom(context, element.children[0])
+        for next_name in element.children[2::2]:
+            # TODO add search_global=True?
+            context_set = context_set.py__getattribute__(next_name, name_context=context)
+        return context_set
+    elif typ == 'eval_input':
+        return eval_node(context, element.children[0])
+    elif typ == 'annassign':
+        return pep0484._evaluate_for_annotation(context, element.children[1])
+    elif typ == 'yield_expr':
+        if len(element.children) and element.children[1].type == 'yield_arg':
+            # Implies that it's a yield from.
+            element = element.children[1].children[1]
+            generators = context.eval_node(element)
+            return _py__stop_iteration_returns(generators)
+
+        # Generator.send() is not implemented.
+        return NO_CONTEXTS
+    else:
+        return eval_or_test(context, element)
+
+
+def eval_trailer(context, base_contexts, trailer):
+    trailer_op, node = trailer.children[:2]
+    if node == ')':  # `arglist` is optional.
+        node = None
+
+    if trailer_op == '[':
+        trailer_op, node, _ = trailer.children
+
+        # TODO It's kind of stupid to cast this from a context set to a set.
+        foo = set(base_contexts)
+        # special case: PEP0484 typing module, see
+        # https://github.com/davidhalter/jedi/issues/663
+        result = ContextSet()
+        for typ in list(foo):
+            if isinstance(typ, (ClassContext, TreeInstance)):
+                typing_module_types = pep0484.py__getitem__(context, typ, node)
+                if typing_module_types is not None:
+                    foo.remove(typ)
+                    result |= typing_module_types
+
+        return result | base_contexts.get_item(
+            eval_subscript_list(context.evaluator, context, node),
+            ContextualizedNode(context, trailer)
+        )
+    else:
+        debug.dbg('eval_trailer: %s in %s', trailer, base_contexts)
+        if trailer_op == '.':
+            return base_contexts.py__getattribute__(
+                name_context=context,
+                name_or_str=node
+            )
+        else:
+            assert trailer_op == '(', 'trailer_op is actually %s' % trailer_op
+            args = arguments.TreeArguments(context.evaluator, context, node, trailer)
+            return base_contexts.execute(args)
+
+
+def eval_atom(context, atom):
+    """
+    Basically to process ``atom`` nodes. The parser sometimes doesn't
+    generate the node (because it has just one child). In that case an atom
+    might be a name or a literal as well.
+    """
+    if atom.type == 'name':
+        # This is the first global lookup.
+        stmt = tree.search_ancestor(
+            atom, 'expr_stmt', 'lambdef'
+        ) or atom
+        if stmt.type == 'lambdef':
+            stmt = atom
+        return context.py__getattribute__(
+            name_or_str=atom,
+            position=stmt.start_pos,
+            search_global=True
+        )
+    elif atom.type == 'keyword':
+        # For False/True/None
+        if atom.value in ('False', 'True', 'None'):
+            return ContextSet(compiled.builtin_from_name(context.evaluator, atom.value))
+        elif atom.value == 'print':
+            # print e.g. could be evaluated like this in Python 2.7
+            return NO_CONTEXTS
+        elif atom.value == 'yield':
+            # Contrary to yield from, yield can just appear alone to return a
+            # value when used with `.send()`.
+            return NO_CONTEXTS
+        assert False, 'Cannot evaluate the keyword %s' % atom
+
+    elif isinstance(atom, tree.Literal):
+        string = context.evaluator.compiled_subprocess.safe_literal_eval(atom.value)
+        return ContextSet(compiled.create_simple_object(context.evaluator, string))
+    elif atom.type == 'strings':
+        # Will be multiple string.
+        context_set = eval_atom(context, atom.children[0])
+        for string in atom.children[1:]:
+            right = eval_atom(context, string)
+            context_set = _eval_comparison(context.evaluator, context, context_set, u'+', right)
+        return context_set
+    else:
+        c = atom.children
+        # Parentheses without commas are not tuples.
+        if c[0] == '(' and not len(c) == 2 \
+                and not(c[1].type == 'testlist_comp' and
+                        len(c[1].children) > 1):
+            return context.eval_node(c[1])
+
+        try:
+            comp_for = c[1].children[1]
+        except (IndexError, AttributeError):
+            pass
+        else:
+            if comp_for == ':':
+                # Dict comprehensions have a colon at the 3rd index.
+                try:
+                    comp_for = c[1].children[3]
+                except IndexError:
+                    pass
+
+            if comp_for.type == 'comp_for':
+                return ContextSet(iterable.comprehension_from_atom(
+                    context.evaluator, context, atom
+                ))
+
+        # It's a dict/list/tuple literal.
+        array_node = c[1]
+        try:
+            array_node_c = array_node.children
+        except AttributeError:
+            array_node_c = []
+        if c[0] == '{' and (array_node == '}' or ':' in array_node_c or
+                            '**' in array_node_c):
+            context = iterable.DictLiteralContext(context.evaluator, context, atom)
+        else:
+            context = iterable.SequenceLiteralContext(context.evaluator, context, atom)
+        return ContextSet(context)
+
+
+@_limit_context_infers
+def eval_expr_stmt(context, stmt, seek_name=None):
+    with recursion.execution_allowed(context.evaluator, stmt) as allowed:
+        # Here we allow list/set to recurse under certain conditions. To make
+        # it possible to resolve stuff like list(set(list(x))), this is
+        # necessary.
+        if not allowed and context.get_root_context() == context.evaluator.builtins_module:
+            try:
+                instance = context.var_args.instance
+            except AttributeError:
+                pass
+            else:
+                if instance.name.string_name in ('list', 'set'):
+                    c = instance.get_first_non_keyword_argument_contexts()
+                    if instance not in c:
+                        allowed = True
+
+        if allowed:
+            return _eval_expr_stmt(context, stmt, seek_name)
+    return NO_CONTEXTS
+
+
+@debug.increase_indent
+def _eval_expr_stmt(context, stmt, seek_name=None):
+    """
+    The starting point of the completion. A statement always owns a call
+    list, which are the calls, that a statement does. In case multiple
+    names are defined in the statement, `seek_name` returns the result for
+    this name.
+
+    :param stmt: A `tree.ExprStmt`.
+    """
+    debug.dbg('eval_expr_stmt %s (%s)', stmt, seek_name)
+    rhs = stmt.get_rhs()
+    context_set = context.eval_node(rhs)
+
+    if seek_name:
+        c_node = ContextualizedName(context, seek_name)
+        context_set = check_tuple_assignments(context.evaluator, c_node, context_set)
+
+    first_operator = next(stmt.yield_operators(), None)
+    if first_operator not in ('=', None) and first_operator.type == 'operator':
+        # `=` is always the last character in aug assignments -> -1
+        operator = copy.copy(first_operator)
+        operator.value = operator.value[:-1]
+        name = stmt.get_defined_names()[0].value
+        left = context.py__getattribute__(
+            name, position=stmt.start_pos, search_global=True)
+
+        for_stmt = tree.search_ancestor(stmt, 'for_stmt')
+        if for_stmt is not None and for_stmt.type == 'for_stmt' and context_set \
+                and parser_utils.for_stmt_defines_one_name(for_stmt):
+            # Iterate through result and add the values, that's possible
+            # only in for loops without clutter, because they are
+            # predictable. Also only do it, if the variable is not a tuple.
+            node = for_stmt.get_testlist()
+            cn = ContextualizedNode(context, node)
+            ordered = list(cn.infer().iterate(cn))
+
+            for lazy_context in ordered:
+                dct = {for_stmt.children[1].value: lazy_context.infer()}
+                with helpers.predefine_names(context, for_stmt, dct):
+                    t = context.eval_node(rhs)
+                    left = _eval_comparison(context.evaluator, context, left, operator, t)
+            context_set = left
+        else:
+            context_set = _eval_comparison(context.evaluator, context, left, operator, context_set)
+    debug.dbg('eval_expr_stmt result %s', context_set)
+    return context_set
+
+
+def eval_or_test(context, or_test):
+    iterator = iter(or_test.children)
+    types = context.eval_node(next(iterator))
+    for operator in iterator:
+        right = next(iterator)
+        if operator.type == 'comp_op':  # not in / is not
+            operator = ' '.join(c.value for c in operator.children)
+
+        # handle lazy evaluation of and/or here.
+        if operator in ('and', 'or'):
+            left_bools = set(left.py__bool__() for left in types)
+            if left_bools == {True}:
+                if operator == 'and':
+                    types = context.eval_node(right)
+            elif left_bools == {False}:
+                if operator != 'and':
+                    types = context.eval_node(right)
+            # Otherwise continue, because of uncertainty.
+        else:
+            types = _eval_comparison(context.evaluator, context, types, operator,
+                                     context.eval_node(right))
+    debug.dbg('eval_or_test types %s', types)
+    return types
+
+
+@iterator_to_context_set
+def eval_factor(context_set, operator):
+    """
+    Calculates `+`, `-`, `~` and `not` prefixes.
+    """
+    for context in context_set:
+        if operator == '-':
+            if is_number(context):
+                yield context.negate()
+        elif operator == 'not':
+            value = context.py__bool__()
+            if value is None:  # Uncertainty.
+                return
+            yield compiled.create_simple_object(context.evaluator, not value)
+        else:
+            yield context
+
+
+def _literals_to_types(evaluator, result):
+    # Changes literals ('a', 1, 1.0, etc) to its type instances (str(),
+    # int(), float(), etc).
+    new_result = NO_CONTEXTS
+    for typ in result:
+        if is_literal(typ):
+            # Literals are only valid as long as the operations are
+            # correct. Otherwise add a value-free instance.
+            cls = compiled.builtin_from_name(evaluator, typ.name.string_name)
+            new_result |= cls.execute_evaluated()
+        else:
+            new_result |= ContextSet(typ)
+    return new_result
+
+
+def _eval_comparison(evaluator, context, left_contexts, operator, right_contexts):
+    if not left_contexts or not right_contexts:
+        # illegal slices e.g. cause left/right_result to be None
+        result = (left_contexts or NO_CONTEXTS) | (right_contexts or NO_CONTEXTS)
+        return _literals_to_types(evaluator, result)
+    else:
+        # I don't think there's a reasonable chance that a string
+        # operation is still correct, once we pass something like six
+        # objects.
+        if len(left_contexts) * len(right_contexts) > 6:
+            return _literals_to_types(evaluator, left_contexts | right_contexts)
+        else:
+            return ContextSet.from_sets(
+                _eval_comparison_part(evaluator, context, left, operator, right)
+                for left in left_contexts
+                for right in right_contexts
+            )
+
+
+def _is_tuple(context):
+    return isinstance(context, iterable.Sequence) and context.array_type == 'tuple'
+
+
+def _is_list(context):
+    return isinstance(context, iterable.Sequence) and context.array_type == 'list'
+
+
+def _bool_to_context(evaluator, bool_):
+    return compiled.builtin_from_name(evaluator, force_unicode(str(bool_)))
+
+
+def _eval_comparison_part(evaluator, context, left, operator, right):
+    l_is_num = is_number(left)
+    r_is_num = is_number(right)
+    if isinstance(operator, unicode):
+        str_operator = operator
+    else:
+        str_operator = force_unicode(str(operator.value))
+
+    if str_operator == '*':
+        # for iterables, ignore * operations
+        if isinstance(left, iterable.Sequence) or is_string(left):
+            return ContextSet(left)
+        elif isinstance(right, iterable.Sequence) or is_string(right):
+            return ContextSet(right)
+    elif str_operator == '+':
+        if l_is_num and r_is_num or is_string(left) and is_string(right):
+            return ContextSet(left.execute_operation(right, str_operator))
+        elif _is_tuple(left) and _is_tuple(right) or _is_list(left) and _is_list(right):
+            return ContextSet(iterable.MergedArray(evaluator, (left, right)))
+    elif str_operator == '-':
+        if l_is_num and r_is_num:
+            return ContextSet(left.execute_operation(right, str_operator))
+    elif str_operator == '%':
+        # With strings and numbers the left type typically remains. Except for
+        # `int() % float()`.
+        return ContextSet(left)
+    elif str_operator in COMPARISON_OPERATORS:
+        if is_compiled(left) and is_compiled(right):
+            # Possible, because the return is not an option. Just compare.
+            try:
+                return ContextSet(left.execute_operation(right, str_operator))
+            except TypeError:
+                # Could be True or False.
+                pass
+        else:
+            if str_operator in ('is', '!=', '==', 'is not'):
+                operation = COMPARISON_OPERATORS[str_operator]
+                bool_ = operation(left, right)
+                return ContextSet(_bool_to_context(evaluator, bool_))
+
+        return ContextSet(_bool_to_context(evaluator, True), _bool_to_context(evaluator, False))
+    elif str_operator == 'in':
+        return NO_CONTEXTS
+
+    def check(obj):
+        """Checks if a Jedi object is either a float or an int."""
+        return isinstance(obj, CompiledInstance) and \
+            obj.name.string_name in ('int', 'float')
+
+    # Static analysis, one is a number, the other one is not.
+    if str_operator in ('+', '-') and l_is_num != r_is_num \
+            and not (check(left) or check(right)):
+        message = "TypeError: unsupported operand type(s) for +: %s and %s"
+        analysis.add(context, 'type-error-operation', operator,
+                     message % (left, right))
+
+    return ContextSet(left, right)
+
+
+def _remove_statements(evaluator, context, stmt, name):
+    """
+    This is the part where statements are being stripped.
+
+    Due to lazy evaluation, statements like a = func; b = a; b() have to be
+    evaluated.
+    """
+    pep0484_contexts = \
+        pep0484.find_type_from_comment_hint_assign(context, stmt, name)
+    if pep0484_contexts:
+        return pep0484_contexts
+
+    return eval_expr_stmt(context, stmt, seek_name=name)
+
+
+def tree_name_to_contexts(evaluator, context, tree_name):
+
+    context_set = ContextSet()
+    module_node = context.get_root_context().tree_node
+    if module_node is not None:
+        names = module_node.get_used_names().get(tree_name.value, [])
+        for name in names:
+            expr_stmt = name.parent
+
+            correct_scope = parser_utils.get_parent_scope(name) == context.tree_node
+
+            if expr_stmt.type == "expr_stmt" and expr_stmt.children[1].type == "annassign" and correct_scope:
+                context_set |= _evaluate_for_annotation(context, expr_stmt.children[1].children[1])
+
+    if context_set:
+        return context_set
+
+    types = []
+    node = tree_name.get_definition(import_name_always=True)
+    if node is None:
+        node = tree_name.parent
+        if node.type == 'global_stmt':
+            context = evaluator.create_context(context, tree_name)
+            finder = NameFinder(evaluator, context, context, tree_name.value)
+            filters = finder.get_filters(search_global=True)
+            # For global_stmt lookups, we only need the first possible scope,
+            # which means the function itself.
+            filters = [next(filters)]
+            return finder.find(filters, attribute_lookup=False)
+        elif node.type not in ('import_from', 'import_name'):
+            raise ValueError("Should not happen. type: %s", node.type)
+
+    typ = node.type
+    if typ == 'for_stmt':
+        types = pep0484.find_type_from_comment_hint_for(context, node, tree_name)
+        if types:
+            return types
+    if typ == 'with_stmt':
+        types = pep0484.find_type_from_comment_hint_with(context, node, tree_name)
+        if types:
+            return types
+
+    if typ in ('for_stmt', 'comp_for'):
+        try:
+            types = context.predefined_names[node][tree_name.value]
+        except KeyError:
+            cn = ContextualizedNode(context, node.children[3])
+            for_types = iterate_contexts(
+                cn.infer(),
+                contextualized_node=cn,
+                is_async=node.parent.type == 'async_stmt',
+            )
+            c_node = ContextualizedName(context, tree_name)
+            types = check_tuple_assignments(evaluator, c_node, for_types)
+    elif typ == 'expr_stmt':
+        types = _remove_statements(evaluator, context, node, tree_name)
+    elif typ == 'with_stmt':
+        context_managers = context.eval_node(node.get_test_node_from_name(tree_name))
+        enter_methods = context_managers.py__getattribute__(u'__enter__')
+        return enter_methods.execute_evaluated()
+    elif typ in ('import_from', 'import_name'):
+        types = imports.infer_import(context, tree_name)
+    elif typ in ('funcdef', 'classdef'):
+        types = _apply_decorators(context, node)
+    elif typ == 'try_stmt':
+        # TODO an exception can also be a tuple. Check for those.
+        # TODO check for types that are not classes and add it to
+        # the static analysis report.
+        exceptions = context.eval_node(tree_name.get_previous_sibling().get_previous_sibling())
+        types = exceptions.execute_evaluated()
+    else:
+        raise ValueError("Should not happen. type: %s" % typ)
+    return types
+
+
+def _apply_decorators(context, node):
+    """
+    Returns the function, that should to be executed in the end.
+    This is also the places where the decorators are processed.
+    """
+    if node.type == 'classdef':
+        decoratee_context = ClassContext(
+            context.evaluator,
+            parent_context=context,
+            tree_node=node
+        )
+    else:
+        decoratee_context = FunctionContext.from_context(context, node)
+    initial = values = ContextSet(decoratee_context)
+    for dec in reversed(node.get_decorators()):
+        debug.dbg('decorator: %s %s', dec, values)
+        dec_values = context.eval_node(dec.children[1])
+        trailer_nodes = dec.children[2:-1]
+        if trailer_nodes:
+            # Create a trailer and evaluate it.
+            trailer = tree.PythonNode('trailer', trailer_nodes)
+            trailer.parent = dec
+            dec_values = eval_trailer(context, dec_values, trailer)
+
+        if not len(dec_values):
+            debug.warning('decorator not found: %s on %s', dec, node)
+            return initial
+
+        values = dec_values.execute(arguments.ValuesArguments([values]))
+        if not len(values):
+            debug.warning('not possible to resolve wrappers found %s', node)
+            return initial
+
+        debug.dbg('decorator end %s', values)
+    return values
+
+
+def check_tuple_assignments(evaluator, contextualized_name, context_set):
+    """
+    Checks if tuples are assigned.
+    """
+    lazy_context = None
+    for index, node in contextualized_name.assignment_indexes():
+        cn = ContextualizedNode(contextualized_name.context, node)
+        iterated = context_set.iterate(cn)
+        for _ in range(index + 1):
+            try:
+                lazy_context = next(iterated)
+            except StopIteration:
+                # We could do this with the default param in next. But this
+                # would allow this loop to run for a very long time if the
+                # index number is high. Therefore break if the loop is
+                # finished.
+                return ContextSet()
+        context_set = lazy_context.infer()
+    return context_set
+
+
+def eval_subscript_list(evaluator, context, index):
+    """
+    Handles slices in subscript nodes.
+    """
+    if index == ':':
+        # Like array[:]
+        return ContextSet(iterable.Slice(context, None, None, None))
+
+    elif index.type == 'subscript' and not index.children[0] == '.':
+        # subscript basically implies a slice operation, except for Python 2's
+        # Ellipsis.
+        # e.g. array[:3]
+        result = []
+        for el in index.children:
+            if el == ':':
+                if not result:
+                    result.append(None)
+            elif el.type == 'sliceop':
+                if len(el.children) == 2:
+                    result.append(el.children[1])
+            else:
+                result.append(el)
+        result += [None] * (3 - len(result))
+
+        return ContextSet(iterable.Slice(context, *result))
+    elif index.type == 'subscriptlist':
+        return NO_CONTEXTS
+
+    # No slices
+    return context.eval_node(index)

jedi/evaluate/sys_path.py

@@ -1,96 +1,35 @@
-import glob
 import os
-import sys
-import imp
-from jedi.evaluate.site import addsitedir
 
-from jedi._compatibility import exec_function, unicode
-from jedi.evaluate.cache import evaluator_function_cache
-from jedi.evaluate.compiled import CompiledObject
-from jedi.evaluate.context import ContextualizedNode
+from jedi._compatibility import unicode, force_unicode, all_suffixes
+from jedi.evaluate.cache import evaluator_method_cache
+from jedi.evaluate.base_context import ContextualizedNode
+from jedi.evaluate.helpers import is_string
+from jedi.common.utils import traverse_parents
+from jedi.parser_utils import get_cached_code_lines
 from jedi import settings
 from jedi import debug
-from jedi import common
 
 
-def get_venv_path(venv):
-    """Get sys.path for specified virtual environment."""
-    sys_path = _get_venv_path_dirs(venv)
-    with common.ignored(ValueError):
-        sys_path.remove('')
-    sys_path = _get_sys_path_with_egglinks(sys_path)
-    # As of now, get_venv_path_dirs does not scan built-in pythonpath and
-    # user-local site-packages, let's approximate them using path from Jedi
-    # interpreter.
-    return sys_path + sys.path
+def _abs_path(module_context, path):
+    if os.path.isabs(path):
+        return path
 
+    module_path = module_context.py__file__()
+    if module_path is None:
+        # In this case we have no idea where we actually are in the file
+        # system.
+        return None
 
-def _get_sys_path_with_egglinks(sys_path):
-    """Find all paths including those referenced by egg-links.
-
-    Egg-link-referenced directories are inserted into path immediately before
-    the directory on which their links were found.  Such directories are not
-    taken into consideration by normal import mechanism, but they are traversed
-    when doing pkg_resources.require.
-    """
-    result = []
-    for p in sys_path:
-        # pkg_resources does not define a specific order for egg-link files
-        # using os.listdir to enumerate them, we're sorting them to have
-        # reproducible tests.
-        for egg_link in sorted(glob.glob(os.path.join(p, '*.egg-link'))):
-            with open(egg_link) as fd:
-                for line in fd:
-                    line = line.strip()
-                    if line:
-                        result.append(os.path.join(p, line))
-                        # pkg_resources package only interprets the first
-                        # non-empty line in egg-link files.
-                        break
-        result.append(p)
-    return result
-
-
-def _get_venv_path_dirs(venv):
-    """Get sys.path for venv without starting up the interpreter."""
-    venv = os.path.abspath(venv)
-    sitedir = _get_venv_sitepackages(venv)
-    sys_path = []
-    addsitedir(sys_path, sitedir)
-    return sys_path
-
-
-def _get_venv_sitepackages(venv):
-    if os.name == 'nt':
-        p = os.path.join(venv, 'lib', 'site-packages')
-    else:
-        p = os.path.join(venv, 'lib', 'python%d.%d' % sys.version_info[:2],
-                         'site-packages')
-    return p
-
-
-def _execute_code(module_path, code):
-    c = "import os; from os.path import *; result=%s"
-    variables = {'__file__': module_path}
-    try:
-        exec_function(c % code, variables)
-    except Exception:
-        debug.warning('sys.path manipulation detected, but failed to evaluate.')
-    else:
-        try:
-            res = variables['result']
-            if isinstance(res, str):
-                return [os.path.abspath(res)]
-        except KeyError:
-            pass
-    return []
+    base_dir = os.path.dirname(module_path)
+    path = force_unicode(path)
+    return os.path.abspath(os.path.join(base_dir, path))
 
 
 def _paths_from_assignment(module_context, expr_stmt):
     """
     Extracts the assigned strings from an assignment that looks as follows::
 
-    >>> sys.path[0:0] = ['module/path', 'another/module/path']
+        sys.path[0:0] = ['module/path', 'another/module/path']
 
     This function is in general pretty tolerant (and therefore 'buggy').
     However, it's not a big issue usually to add more paths to Jedi's sys_path,
@@ -108,8 +47,8 @@ def _paths_from_assignment(module_context, expr_stmt):
             assert trailer.children[0] == '.' and trailer.children[1].value == 'path'
             # TODO Essentially we're not checking details on sys.path
             # manipulation. Both assigment of the sys.path and changing/adding
-            # parts of the sys.path are the same: They get added to the current
-            # sys.path.
+            # parts of the sys.path are the same: They get added to the end of
+            # the current sys.path.
             """
             execution = c[2]
             assert execution.children[0] == '['
@@ -120,34 +59,40 @@ def _paths_from_assignment(module_context, expr_stmt):
         except AssertionError:
             continue
 
-        from jedi.evaluate.iterable import py__iter__
-        from jedi.evaluate.precedence import is_string
         cn = ContextualizedNode(module_context.create_context(expr_stmt), expr_stmt)
-        for lazy_context in py__iter__(module_context.evaluator, cn.infer(), cn):
+        for lazy_context in cn.infer().iterate(cn):
             for context in lazy_context.infer():
                 if is_string(context):
-                    yield context.obj
+                    abs_path = _abs_path(module_context, context.get_safe_value())
+                    if abs_path is not None:
+                        yield abs_path
 
 
-def _paths_from_list_modifications(module_path, trailer1, trailer2):
+def _paths_from_list_modifications(module_context, trailer1, trailer2):
     """ extract the path from either "sys.path.append" or "sys.path.insert" """
     # Guarantee that both are trailers, the first one a name and the second one
     # a function execution with at least one param.
     if not (trailer1.type == 'trailer' and trailer1.children[0] == '.'
             and trailer2.type == 'trailer' and trailer2.children[0] == '('
             and len(trailer2.children) == 3):
-        return []
+        return
 
     name = trailer1.children[1].value
     if name not in ['insert', 'append']:
-        return []
+        return
     arg = trailer2.children[1]
     if name == 'insert' and len(arg.children) in (3, 4):  # Possible trailing comma.
         arg = arg.children[2]
-    return _execute_code(module_path, arg.get_code())
+
+    for context in module_context.create_context(arg).eval_node(arg):
+        if is_string(context):
+            abs_path = _abs_path(module_context, context.get_safe_value())
+            if abs_path is not None:
+                yield abs_path
 
 
-def _check_module(module_context):
+@evaluator_method_cache(default=[])
+def check_sys_path_modifications(module_context):
     """
     Detect sys.path modifications within module.
     """
@@ -162,10 +107,10 @@ def _check_module(module_context):
                     if n.type == 'name' and n.value == 'path':
                         yield name, power
 
-    sys_path = list(module_context.evaluator.sys_path)  # copy
-    if isinstance(module_context, CompiledObject):
-        return sys_path
+    if module_context.tree_node is None:
+        return []
 
+    added = []
     try:
         possible_names = module_context.tree_node.get_used_names()['path']
     except KeyError:
@@ -174,44 +119,29 @@ def _check_module(module_context):
         for name, power in get_sys_path_powers(possible_names):
             expr_stmt = power.parent
             if len(power.children) >= 4:
-                sys_path.extend(
+                added.extend(
                     _paths_from_list_modifications(
-                        module_context.py__file__(), *power.children[2:4]
+                        module_context, *power.children[2:4]
                     )
                 )
             elif expr_stmt is not None and expr_stmt.type == 'expr_stmt':
-                sys_path.extend(_paths_from_assignment(module_context, expr_stmt))
-    return sys_path
+                added.extend(_paths_from_assignment(module_context, expr_stmt))
+    return added
 
 
-@evaluator_function_cache(default=[])
-def sys_path_with_modifications(evaluator, module_context):
-    path = module_context.py__file__()
-    if path is None:
-        # Support for modules without a path is bad, therefore return the
-        # normal path.
-        return list(evaluator.sys_path)
-
-    curdir = os.path.abspath(os.curdir)
-    #TODO why do we need a chdir?
-    with common.ignored(OSError):
-        os.chdir(os.path.dirname(path))
-
+def discover_buildout_paths(evaluator, script_path):
     buildout_script_paths = set()
 
-    result = _check_module(module_context)
-    result += _detect_django_path(path)
-    for buildout_script_path in _get_buildout_script_paths(path):
+    for buildout_script_path in _get_buildout_script_paths(script_path):
         for path in _get_paths_from_buildout_script(evaluator, buildout_script_path):
             buildout_script_paths.add(path)
-    # cleanup, back to old directory
-    os.chdir(curdir)
-    return list(result) + list(buildout_script_paths)
+
+    return buildout_script_paths
 
 
 def _get_paths_from_buildout_script(evaluator, buildout_script_path):
     try:
-        module_node = evaluator.grammar.parse(
+        module_node = evaluator.parse(
             path=buildout_script_path,
             cache=True,
             cache_path=settings.cache_directory
@@ -220,17 +150,12 @@ def _get_paths_from_buildout_script(evaluator, buildout_script_path):
         debug.warning('Error trying to read buildout_script: %s', buildout_script_path)
         return
 
-    from jedi.evaluate.representation import ModuleContext
-    for path in _check_module(ModuleContext(evaluator, module_node, buildout_script_path)):
-        yield path
-
-
-def traverse_parents(path):
-    while True:
-        new = os.path.dirname(path)
-        if new == path:
-            return
-        path = new
+    from jedi.evaluate.context import ModuleContext
+    module = ModuleContext(
+        evaluator, module_node, buildout_script_path,
+        code_lines=get_cached_code_lines(evaluator.grammar, buildout_script_path),
+    )
+    for path in check_sys_path_modifications(module):
         yield path
 
 
@@ -241,55 +166,42 @@ def _get_parent_dir_with_file(path, filename):
     return None
 
 
-def _detect_django_path(module_path):
-    """ Detects the path of the very well known Django library (if used) """
-    result = []
-
-    for parent in traverse_parents(module_path):
-        with common.ignored(IOError):
-            with open(parent + os.path.sep + 'manage.py'):
-                debug.dbg('Found django path: %s', module_path)
-                result.append(parent)
-    return result
-
-
-def _get_buildout_script_paths(module_path):
+def _get_buildout_script_paths(search_path):
     """
     if there is a 'buildout.cfg' file in one of the parent directories of the
     given module it will return a list of all files in the buildout bin
     directory that look like python files.
 
-    :param module_path: absolute path to the module.
-    :type module_path: str
+    :param search_path: absolute path to the module.
+    :type search_path: str
     """
-    project_root = _get_parent_dir_with_file(module_path, 'buildout.cfg')
+    project_root = _get_parent_dir_with_file(search_path, 'buildout.cfg')
     if not project_root:
-        return []
+        return
     bin_path = os.path.join(project_root, 'bin')
     if not os.path.exists(bin_path):
-        return []
-    extra_module_paths = []
+        return
+
     for filename in os.listdir(bin_path):
         try:
             filepath = os.path.join(bin_path, filename)
             with open(filepath, 'r') as f:
                 firstline = f.readline()
                 if firstline.startswith('#!') and 'python' in firstline:
-                    extra_module_paths.append(filepath)
+                    yield filepath
         except (UnicodeDecodeError, IOError) as e:
-            # Probably a binary file; permission error or race cond. because file got deleted
-            # ignore
+            # Probably a binary file; permission error or race cond. because
+            # file got deleted. Ignore it.
             debug.warning(unicode(e))
             continue
-    return extra_module_paths
 
 
 def dotted_path_in_sys_path(sys_path, module_path):
     """
-    Returns the dotted path inside a sys.path.
+    Returns the dotted path inside a sys.path as a list of names.
     """
     # First remove the suffix.
-    for suffix, _, _ in imp.get_suffixes():
+    for suffix in all_suffixes():
         if module_path.endswith(suffix):
             module_path = module_path[:-len(suffix)]
         break
@@ -309,6 +221,6 @@ def dotted_path_in_sys_path(sys_path, module_path):
                 for string in split:
                     if not string or '.' in string:
                         return None
-                return '.'.join(split)
+                return split
 
     return None

jedi/evaluate/usages.py

@@ -0,0 +1,62 @@
+from jedi.evaluate import imports
+from jedi.evaluate.filters import TreeNameDefinition
+from jedi.evaluate.context import ModuleContext
+
+
+def _resolve_names(definition_names, avoid_names=()):
+    for name in definition_names:
+        if name in avoid_names:
+            # Avoiding recursions here, because goto on a module name lands
+            # on the same module.
+            continue
+
+        if not isinstance(name, imports.SubModuleName):
+            # SubModuleNames are not actually existing names but created
+            # names when importing something like `import foo.bar.baz`.
+            yield name
+
+        if name.api_type == 'module':
+            for name in _resolve_names(name.goto(), definition_names):
+                yield name
+
+
+def _dictionarize(names):
+    return dict(
+        (n if n.tree_name is None else n.tree_name, n)
+        for n in names
+    )
+
+
+def _find_names(module_context, tree_name):
+    context = module_context.create_context(tree_name)
+    name = TreeNameDefinition(context, tree_name)
+    found_names = set(name.goto())
+    found_names.add(name)
+    return _dictionarize(_resolve_names(found_names))
+
+
+def usages(module_context, tree_name):
+    search_name = tree_name.value
+    found_names = _find_names(module_context, tree_name)
+    modules = set(d.get_root_context() for d in found_names.values())
+    modules = set(m for m in modules if isinstance(m, ModuleContext))
+
+    non_matching_usage_maps = {}
+    for m in imports.get_modules_containing_name(module_context.evaluator, modules, search_name):
+        for name_leaf in m.tree_node.get_used_names().get(search_name, []):
+            new = _find_names(m, name_leaf)
+            if any(tree_name in found_names for tree_name in new):
+                found_names.update(new)
+                for tree_name in new:
+                    for dct in non_matching_usage_maps.get(tree_name, []):
+                        # A usage that was previously searched for matches with
+                        # a now found name. Merge.
+                        found_names.update(dct)
+                    try:
+                        del non_matching_usage_maps[tree_name]
+                    except KeyError:
+                        pass
+            else:
+                for name in new:
+                    non_matching_usage_maps.setdefault(name, []).append(new)
+    return found_names.values()

jedi/evaluate/utils.py

@@ -2,9 +2,28 @@
 import sys
 import contextlib
 import functools
+import re
+import os
 
 from jedi._compatibility import reraise
-from jedi import settings
+
+
+_sep = os.path.sep
+if os.path.altsep is not None:
+    _sep += os.path.altsep
+_path_re = re.compile(r'(?:\.[^{0}]+|[{0}]__init__\.py)$'.format(re.escape(_sep)))
+del _sep
+
+
+def to_list(func):
+    def wrapper(*args, **kwargs):
+        return list(func(*args, **kwargs))
+    return wrapper
+
+
+def unite(iterable):
+    """Turns a two dimensional array into a one dimensional."""
+    return set(typ for types in iterable for typ in types)
 
 
 class UncaughtAttributeError(Exception):
@@ -78,16 +97,6 @@ class PushBackIterator(object):
         return self.current
 
 
-def indent_block(text, indention='    '):
-    """This function indents a text block with a default of four spaces."""
-    temp = ''
-    while text and text[-1] == '\n':
-        temp += text[-1]
-        text = text[:-1]
-    lines = text.split('\n')
-    return '\n'.join(map(lambda s: indention + s, lines)) + temp
-
-
 @contextlib.contextmanager
 def ignored(*exceptions):
     """
@@ -100,12 +109,11 @@ def ignored(*exceptions):
         pass
 
 
-def unite(iterable):
-    """Turns a two dimensional array into a one dimensional."""
-    return set(typ for types in iterable for typ in types)
-
-
-def to_list(func):
-    def wrapper(*args, **kwargs):
-        return list(func(*args, **kwargs))
-    return wrapper
+def indent_block(text, indention='    '):
+    """This function indents a text block with a default of four spaces."""
+    temp = ''
+    while text and text[-1] == '\n':
+        temp += text[-1]
+        text = text[:-1]
+    lines = text.split('\n')
+    return '\n'.join(map(lambda s: indention + s, lines)) + temp

jedi/parser_utils.py

@@ -1,14 +1,16 @@
+import re
 import textwrap
 from inspect import cleandoc
 
-from jedi._compatibility import literal_eval, is_py3
 from parso.python import tree
+from parso.cache import parser_cache
 
-_EXECUTE_NODES = set([
-    'funcdef', 'classdef', 'import_from', 'import_name', 'test', 'or_test',
-    'and_test', 'not_test', 'comparison', 'expr', 'xor_expr', 'and_expr',
-    'shift_expr', 'arith_expr', 'atom_expr', 'term', 'factor', 'power', 'atom'
-])
+from jedi._compatibility import literal_eval, force_unicode
+
+_EXECUTE_NODES = {'funcdef', 'classdef', 'import_from', 'import_name', 'test',
+                  'or_test', 'and_test', 'not_test', 'comparison', 'expr',
+                  'xor_expr', 'and_expr', 'shift_expr', 'arith_expr',
+                  'atom_expr', 'term', 'factor', 'power', 'atom'}
 
 _FLOW_KEYWORDS = (
     'try', 'except', 'finally', 'else', 'if', 'elif', 'with', 'for', 'while'
@@ -87,10 +89,12 @@ def get_flow_branch_keyword(flow_node, node):
             keyword = first_leaf
     return 0
 
+
 def get_statement_of_position(node, pos):
     for c in node.children:
         if c.start_pos <= pos <= c.end_pos:
-            if c.type not in ('decorated', 'simple_stmt', 'suite') \
+            if c.type not in ('decorated', 'simple_stmt', 'suite',
+                              'async_stmt', 'async_funcdef') \
                     and not isinstance(c, (tree.Flow, tree.ClassOrFunc)):
                 return c
             else:
@@ -112,10 +116,7 @@ def clean_scope_docstring(scope_node):
         cleaned = cleandoc(safe_literal_eval(node.value))
         # Since we want the docstr output to be always unicode, just
         # force it.
-        if is_py3 or isinstance(cleaned, unicode):
-            return cleaned
-        else:
-            return unicode(cleaned, 'UTF-8', 'replace')
+        return force_unicode(cleaned)
     return ''
 
 
@@ -158,7 +159,12 @@ def get_call_signature(funcdef, width=72, call_string=None):
         p = '(' + ''.join(param.get_code() for param in funcdef.get_params()).strip() + ')'
     else:
         p = funcdef.children[2].get_code()
-    code = call_string + p
+    p = re.sub(r'\s+', ' ', p)
+    if funcdef.annotation:
+        rtype = " ->" + funcdef.annotation.get_code()
+    else:
+        rtype = ""
+    code = call_string + p + rtype
 
     return '\n'.join(textwrap.wrap(code, width))
 
@@ -179,6 +185,8 @@ def get_doc_with_call_signature(scope_node):
     doc = clean_scope_docstring(scope_node)
     if call_signature is None:
         return doc
+    if not doc:
+        return call_signature
     return '%s\n\n%s' % (call_signature, doc)
 
 
@@ -205,6 +213,9 @@ def get_following_comment_same_line(node):
             whitespace = node.children[5].get_first_leaf().prefix
         elif node.type == 'with_stmt':
             whitespace = node.children[3].get_first_leaf().prefix
+        elif node.type == 'funcdef':
+            # actually on the next line
+            whitespace = node.children[4].get_first_leaf().get_next_leaf().prefix
         else:
             whitespace = node.get_last_leaf().get_next_leaf().prefix
     except AttributeError:
@@ -239,3 +250,11 @@ def get_parent_scope(node, include_flows=False):
             break
         scope = scope.parent
     return scope
+
+
+def get_cached_code_lines(grammar, path):
+    """
+    Basically access the cached code lines in parso. This is not the nicest way
+    to do this, but we avoid splitting all the lines again.
+    """
+    return parser_cache[grammar._hashed][path].lines

jedi/refactoring.py

@@ -1,11 +1,14 @@
 """
+THIS is not in active development, please check
+https://github.com/davidhalter/jedi/issues/667 first before editing.
+
 Introduce some basic refactoring functions to |jedi|. This module is still in a
 very early development stage and needs much testing and improvement.
 
 .. warning:: I won't do too much here, but if anyone wants to step in, please
              do. Refactoring is none of my priorities
 
-It uses the |jedi| `API <plugin-api.html>`_ and supports currently the
+It uses the |jedi| `API <api.html>`_ and supports currently the
 following functions (sometimes bug-prone):
 
 - rename
@@ -14,7 +17,6 @@ following functions (sometimes bug-prone):
 """
 import difflib
 
-from jedi import common
 from parso import python_bytes_to_unicode, split_lines
 from jedi.evaluate import helpers
 
@@ -51,9 +53,8 @@ class Refactoring(object):
 
 def rename(script, new_name):
     """ The `args` / `kwargs` params are the same as in `api.Script`.
-    :param operation: The refactoring operation to execute.
-    :type operation: str
-    :type source: str
+    :param new_name: The new name of the script.
+    :param script: The source Script object.
     :return: list of changed lines/changed files
     """
     return Refactoring(_rename(script.usages(), new_name))
@@ -106,11 +107,12 @@ def extract(script, new_name):
 
     user_stmt = script._parser.user_stmt()
 
-    # TODO care for multiline extracts
+    # TODO care for multi-line extracts
     dct = {}
     if user_stmt:
         pos = script._pos
         line_index = pos[0] - 1
+        # Be careful here. 'array_for_pos' does not exist in 'helpers'.
         arr, index = helpers.array_for_pos(user_stmt, pos)
         if arr is not None:
             start_pos = arr[index].start_pos
@@ -121,7 +123,7 @@ def extract(script, new_name):
             start_line = new_lines[start_pos[0] - 1]
             text = start_line[start_pos[1]:e]
             for l in range(start_pos[0], end_pos[0] - 1):
-                text += '\n' + l
+                text += '\n' + str(l)
             if e is None:
                 end_line = new_lines[end_pos[0] - 1]
                 text += '\n' + end_line[:end_pos[1]]
@@ -141,7 +143,7 @@ def extract(script, new_name):
             new_lines[start_pos[0] - 1] = start_line
             new_lines[start_pos[0]:end_pos[0] - 1] = []
 
-            # add parentheses in multiline case
+            # add parentheses in multi-line case
             open_brackets = ['(', '[', '{']
             close_brackets = [')', ']', '}']
             if '\n' in text and not (text[0] in open_brackets and text[-1] ==
@@ -165,38 +167,37 @@ def inline(script):
     dct = {}
 
     definitions = script.goto_assignments()
-    with common.ignored(AssertionError):
-        assert len(definitions) == 1
-        stmt = definitions[0]._definition
-        usages = script.usages()
-        inlines = [r for r in usages
-                   if not stmt.start_pos <= (r.line, r.column) <= stmt.end_pos]
-        inlines = sorted(inlines, key=lambda x: (x.module_path, x.line, x.column),
-                         reverse=True)
-        expression_list = stmt.expression_list()
-        # don't allow multiline refactorings for now.
-        assert stmt.start_pos[0] == stmt.end_pos[0]
-        index = stmt.start_pos[0] - 1
+    assert len(definitions) == 1
+    stmt = definitions[0]._definition
+    usages = script.usages()
+    inlines = [r for r in usages
+               if not stmt.start_pos <= (r.line, r.column) <= stmt.end_pos]
+    inlines = sorted(inlines, key=lambda x: (x.module_path, x.line, x.column),
+                     reverse=True)
+    expression_list = stmt.expression_list()
+    # don't allow multi-line refactorings for now.
+    assert stmt.start_pos[0] == stmt.end_pos[0]
+    index = stmt.start_pos[0] - 1
 
-        line = new_lines[index]
-        replace_str = line[expression_list[0].start_pos[1]:stmt.end_pos[1] + 1]
-        replace_str = replace_str.strip()
-        # tuples need parentheses
-        if expression_list and isinstance(expression_list[0], pr.Array):
-            arr = expression_list[0]
-            if replace_str[0] not in ['(', '[', '{'] and len(arr) > 1:
-                replace_str = '(%s)' % replace_str
+    line = new_lines[index]
+    replace_str = line[expression_list[0].start_pos[1]:stmt.end_pos[1] + 1]
+    replace_str = replace_str.strip()
+    # tuples need parentheses
+    if expression_list and isinstance(expression_list[0], pr.Array):
+        arr = expression_list[0]
+        if replace_str[0] not in ['(', '[', '{'] and len(arr) > 1:
+            replace_str = '(%s)' % replace_str
 
-        # if it's the only assignment, remove the statement
-        if len(stmt.get_defined_names()) == 1:
-            line = line[:stmt.start_pos[1]] + line[stmt.end_pos[1]:]
+    # if it's the only assignment, remove the statement
+    if len(stmt.get_defined_names()) == 1:
+        line = line[:stmt.start_pos[1]] + line[stmt.end_pos[1]:]
 
-        dct = _rename(inlines, replace_str)
-        # remove the empty line
-        new_lines = dct[script.path][2]
-        if line.strip():
-            new_lines[index] = line
-        else:
-            new_lines.pop(index)
+    dct = _rename(inlines, replace_str)
+    # remove the empty line
+    new_lines = dct[script.path][2]
+    if line.strip():
+        new_lines[index] = line
+    else:
+        new_lines.pop(index)
 
     return Refactoring(dct)

jedi/settings.py

@@ -144,7 +144,8 @@ Check for `isinstance` and other information to infer a type.
 """
 
 auto_import_modules = [
-    'hashlib',  # setattr
+    'hashlib',  # hashlib is mostly using setattr, which jedi doesn't understand
+    'gi',  # This third-party repository (GTK stuff) doesn't really work with jedi
 ]
 """
 Modules that are not analyzed but imported, although they contain Python code.

jedi/utils.py

@@ -89,12 +89,13 @@ def setup_readline(namespace_module=__main__):
                     lines = split_lines(text)
                     position = (len(lines), len(lines[-1]))
                     name = get_on_completion_name(
-                        interpreter._get_module_node(),
+                        interpreter._module_node,
                         lines,
                         position
                     )
                     before = text[:len(text) - len(name)]
                     completions = interpreter.completions()
+                    logging.debug("REPL completions: %s", completions)
                 except:
                     logging.error("REPL Completion error:\n" + traceback.format_exc())
                     raise
@@ -108,6 +109,11 @@ def setup_readline(namespace_module=__main__):
                 return None
 
     try:
+        # Need to import this one as well to make sure it's executed before
+        # this code. This didn't use to be an issue until 3.3. Starting with
+        # 3.4 this is different, it always overwrites the completer if it's not
+        # already imported here.
+        import rlcompleter  # noqa: F401
         import readline
     except ImportError:
         print("Jedi: Module readline not available.")
@@ -132,5 +138,5 @@ def version_info():
     """
     Version = namedtuple('Version', 'major, minor, micro')
     from jedi import __version__
-    tupl = re.findall('[a-z]+|\d+', __version__)
+    tupl = re.findall(r'[a-z]+|\d+', __version__)
     return Version(*[x if i == 3 else int(x) for i, x in enumerate(tupl)])

pytest.ini

@@ -2,7 +2,9 @@
 addopts = --doctest-modules
 
 # Ignore broken files in blackbox test directories
-norecursedirs = .* docs completion refactor absolute_import namespace_package scripts extensions speed static_analysis not_in_sys_path buildout_project sample_venvs init_extension_module simple_import
+norecursedirs = .* docs completion refactor absolute_import namespace_package
+    scripts extensions speed static_analysis not_in_sys_path
+    sample_venvs init_extension_module simple_import
 
 # Activate `clean_jedi_cache` fixture for all tests.  This should be
 # fine as long as we are using `clean_jedi_cache` as a session scoped

requirements.txt

@@ -1 +1 @@
-parso==0.1.0
+parso>=0.3.0

scripts/wx_check.py

@@ -8,7 +8,7 @@ like a typical big Python modules. A mix between a lot of different Python
 things.
 
 You can view a markup version of it here:
-http://svn.wxwidgets.org/viewvc/wx/wxPython/trunk/src/gtk/_core.py?view=markup
+https://github.com/wxWidgets/wxPython/blob/master/src/gtk/_core.py
 """
 
 import resource
@@ -30,11 +30,12 @@ import jedi
 def process_memory():
     """
     In kB according to
-    http://stackoverflow.com/questions/938733/total-memory-used-by-python-process
+    https://stackoverflow.com/questions/938733/total-memory-used-by-python-process
     """
     return resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
 
-uri = 'http://svn.wxwidgets.org/viewvc/wx/wxPython/trunk/src/gtk/_core.py?revision=74740&content-type=text%2Fplain&view=co'
+
+uri = 'https://raw.githubusercontent.com/wxWidgets/wxPython/master/src/gtk/_core.py'
 
 wx_core = urllib2.urlopen(uri).read()
 

setup.cfg

@@ -1,2 +1,8 @@
 [bdist_wheel]
 universal=1
+
+[flake8]
+max-line-length = 100
+ignore =
+  # do not use bare 'except'
+  E722,

setup.py

@@ -1,20 +1,18 @@
 #!/usr/bin/env python
 
-from setuptools import setup
+from setuptools import setup, find_packages
 
 import ast
 
-
 __AUTHOR__ = 'David Halter'
 __AUTHOR_EMAIL__ = 'davidhalter88@gmail.com'
 
 # Get the version from within jedi. It's defined in exactly one place now.
 with open('jedi/__init__.py') as f:
     tree = ast.parse(f.read())
-version = tree.body[1].value.s
+version = tree.body[int(not hasattr(tree, 'docstring'))].value.s
 
 readme = open('README.rst').read() + '\n\n' + open('CHANGELOG.rst').read()
-packages = ['jedi', 'jedi.evaluate', 'jedi.evaluate.compiled', 'jedi.api']
 with open('requirements.txt') as f:
     install_requires = f.read().splitlines()
 
@@ -30,8 +28,18 @@ setup(name='jedi',
       license='MIT',
       keywords='python completion refactoring vim',
       long_description=readme,
-      packages=packages,
+      packages=find_packages(exclude=['test', 'test.*']),
+      python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*',
       install_requires=install_requires,
+      extras_require={
+          'testing': [
+              'pytest>=3.1.0',
+              # docopt for sith doctests
+              'docopt',
+              # coloroma for colored debug output
+              'colorama',
+          ],
+      },
       package_data={'jedi': ['evaluate/compiled/fake/*.pym']},
       platforms=['any'],
       classifiers=[
@@ -41,12 +49,12 @@ setup(name='jedi',
           'License :: OSI Approved :: MIT License',
           'Operating System :: OS Independent',
           'Programming Language :: Python :: 2',
-          'Programming Language :: Python :: 2.6',
           'Programming Language :: Python :: 2.7',
           'Programming Language :: Python :: 3',
-          'Programming Language :: Python :: 3.3',
           'Programming Language :: Python :: 3.4',
           'Programming Language :: Python :: 3.5',
+          'Programming Language :: Python :: 3.6',
+          'Programming Language :: Python :: 3.7',
           'Topic :: Software Development :: Libraries :: Python Modules',
           'Topic :: Text Editors :: Integrated Development Environments (IDE)',
           'Topic :: Utilities',

sith.py

@@ -127,8 +127,6 @@ class TestCase(object):
                 print("{path}: Line {line} column {column}".format(**self.__dict__))
                 self.show_location(self.line, self.column)
                 self.show_operation()
-        except jedi.NotFoundError:
-            pass
         except Exception:
             self.traceback = traceback.format_exc()
             if record is not None:

test/blabla_test_documentation.py

@@ -1,6 +1,4 @@
-# todo probably remove test_integration_keyword
-
-def test_keyword_doc():
+def test_keyword_doc(Script):
     r = list(Script("or", 1, 1).goto_definitions())
     assert len(r) == 1
     assert len(r[0].doc) > 100
@@ -14,17 +12,18 @@ def test_keyword_doc():
     assert k.doc.startswith(imp_start)
 
 
-def test_blablabla():
+def test_blablabla(Script):
     defs = Script("import").goto_definitions()
     assert len(defs) == 1 and [1 for d in defs if d.doc]
     # unrelated to #44
 
 
-def test_operator_doc(self):
+def test_operator_doc(Script):
     r = list(Script("a == b", 1, 3).goto_definitions())
     assert len(r) == 1
     assert len(r[0].doc) > 100
 
-def test_lambda():
+
+def test_lambda(Script):
     defs = Script('lambda x: x', column=0).goto_definitions()
     assert [d.type for d in defs] == ['keyword']

test/completion/__init__.py

@@ -1,3 +1,8 @@
 """ needed for some modules to test against packages. """
 
 some_variable = 1
+
+
+from . import imports
+#? int()
+imports.relative()

test/completion/arrays.py

@@ -29,6 +29,9 @@ b = [6,7]
 
 #? int()
 b[8-7]
+# Something unreasonable:
+#?
+b['']
 
 # -----------------
 # Slices
@@ -42,6 +45,8 @@ b[int():]
 #? list()
 b[:]
 
+#?
+b[:, 1]
 
 class _StrangeSlice():
     def __getitem__(self, sliced):
@@ -204,6 +209,9 @@ g
 dic2 = {'asdf': 3, 'b': 'str'}
 #? int()
 dic2['asdf']
+# TODO for now get doesn't work properly when used with a literal.
+#? None
+dic2.get('asdf')
 
 # string literal
 #? int()
@@ -256,6 +264,17 @@ for x in {1: 3.0, '': 1j}:
     #? int() str()
     x
 
+#? ['__iter__']
+dict().values().__iter__
+
+d = dict(a=3, b='')
+#? int() str()
+d.values()[0]
+#? int()
+d['a']
+#? int() None
+d.get('a')
+
 # -----------------
 # with variable as index
 # -----------------
@@ -387,6 +406,18 @@ for x in [1] + ['']:
     #? int() str()
     x
 
+# -----------------
+# Potential Recursion Issues
+# -----------------
+class X():
+    def y(self):
+        self.a = [1]
+
+    def x(self):
+        self.a = list(self.a)
+        #? int()
+        self.a[0]
+
 # -----------------
 # For loops with attribute assignment.
 # -----------------
@@ -403,12 +434,10 @@ def test_func():
     x
 
 
-# python >= 2.7
-# Set literals are not valid in 2.6.
 #? int()
 tuple({1})[0]
 
-# python >= 3.3
+# python >= 3.4
 # -----------------
 # PEP 3132 Extended Iterable Unpacking (star unpacking)
 # -----------------

test/completion/async_.py

@@ -5,21 +5,80 @@ Currently we're not supporting completion of them, but they should at least not
 raise errors or return extremely strange results.
 """
 
-async def x():
-    argh = await x()
-    #? 
-    argh
-    return 2
+# python >= 3.5
 
-#? int()
-x()
+async def x():
+    return 1
+
+#? []
+x.cr_awai
+
+#? ['cr_await']
+x().cr_awai
 
 a = await x()
-#?
+#? int()
 a
 
+async def y():
+    argh = await x()
+    #? int()
+    argh
+    #? ['__next__']
+    x().__await__().__next
+    return 2
 
 async def x2():
     async with open('asdf') as f:
         #? ['readlines']
         f.readlines
+
+class A():
+    @staticmethod
+    async def b(c=1, d=2):
+        return 1
+
+#! 9 ['def b']
+await A.b()
+
+#! 11 ['param d=2']
+await A.b(d=3)
+
+class Awaitable:
+    def __await__(self):
+        yield None
+        return ''
+
+async def awaitable_test():
+    foo = await Awaitable()
+    #? str()
+    foo
+
+# python >= 3.6
+
+async def asgen():
+    yield 1
+    await asyncio.sleep(0)
+    yield 2
+
+async def wrapper():
+    #? int()
+    [x async for x in asgen()][0]
+
+    async for y in asgen():
+        #? int()
+        y
+
+#? ['__anext__']
+asgen().__ane
+#? []
+asgen().mro
+
+
+# Normal completion (#1092)
+normal_var1 = 42
+
+async def foo():
+    normal_var2 = False
+    #? ['normal_var1', 'normal_var2']
+    normal_var

test/completion/basic.py

@@ -18,6 +18,13 @@ int(str)
 str..
 #? []
 a(0):.
+#? 2 ['and', 'or', 'if', 'is', 'in', 'not']
+0x0
+#? ['and', 'or', 'if', 'is', 'in', 'not']
+1j
+x = None()
+#?
+x
 
 # -----------------
 # if/else/elif
@@ -147,6 +154,9 @@ def global_define():
 #? int()
 global_var_in_func
 
+#? ['global_var_in_func']
+global_var_in_f
+
 
 def funct1():
     # From issue #610
@@ -158,6 +168,17 @@ def funct2():
     global_dict_var
 
 
+global_var_predefined = None
+
+def init_global_var_predefined():
+    global global_var_predefined
+    if global_var_predefined is None:
+        global_var_predefined = 3
+
+#? int() None
+global_var_predefined
+
+
 # -----------------
 # within docstrs
 # -----------------
@@ -286,8 +307,6 @@ with open('') as f:
         #? str()
         line
 
-# Nested with statements don't exist in Python 2.6.
-# python >= 2.7
 with open('') as f1, open('') as f2:
     #? ['closed']
     f1.closed

test/completion/classes.py

@@ -215,6 +215,32 @@ class Dude(classgetter()):
         #? ['shout']
         self.s
 
+# -----------------
+# multiple inheritance # 1071
+# -----------------
+
+class FactorMixin(object):
+    FACTOR_1 = 0.1
+
+class Calc(object):
+    def sum(self, a, b):
+        self.xxx = 3
+        return a + b
+
+class BetterCalc(Calc, FactorMixin):
+    def multiply_factor(self, a):
+        return a * self.FACTOR_1
+
+calc = BetterCalc()
+#? ['sum']
+calc.sum
+#? ['multiply_factor']
+calc.multip
+#? ['FACTOR_1']
+calc.FACTOR_1
+#? ['xxx']
+calc.xxx
+
 # -----------------
 # __call__
 # -----------------
@@ -391,6 +417,9 @@ class PrivateVar():
     def __private_func(self):
         return 1
 
+    #? int()
+    __private_func()
+
     def wrap_private(self):
         return self.__private_func()
 #? []
@@ -399,6 +428,8 @@ PrivateVar().__var
 PrivateVar().__var
 #? []
 PrivateVar().__private_func
+#? []
+PrivateVar.__private_func
 #? int()
 PrivateVar().wrap_private()
 
@@ -422,17 +453,18 @@ class Super(object):
     a = 3
     def return_sup(self):
         return 1
+SuperCopy = Super
 
 class TestSuper(Super):
     #?
     super()
     def test(self):
-        #? Super()
+        #? SuperCopy()
         super()
         #? ['a']
         super().a
         if 1:
-            #? Super()
+            #? SuperCopy()
             super()
         def a():
             #?
@@ -446,6 +478,17 @@ class TestSuper(Super):
 TestSuper().return_sup()
 
 
+Super = 3
+
+class Foo():
+    def foo(self):
+        return 1
+# Somehow overwriting the same name caused problems (#1044)
+class Foo(Foo):
+    def foo(self):
+        #? int()
+        super().foo()
+
 # -----------------
 # if flow at class level
 # -----------------