typo

* Sort completions by input resemblance.

Fixes #2017

* Clean code

.coveragerc

@@ -4,7 +4,6 @@ omit =
     jedi/inference/compiled/subprocess/__main__.py
     jedi/__main__.py
     # For now this is not being used.
-    jedi/refactoring.py
 
 [report]
 # Regexes for lines to exclude from consideration

.editorconfig

@@ -0,0 +1,14 @@
+root = true
+
+[*]
+charset = utf-8
+end_of_line = lf
+indent_style = space
+insert_final_newline = true
+trim_trailing_whitespace = true
+
+[*.py]
+indent_size = 4
+
+[*.md]
+indent_size = 2

.gitattributes

@@ -0,0 +1,10 @@
+# all end-of-lines are normalized to LF when written to the repository
+# https://git-scm.com/docs/gitattributes#_text
+* text=auto
+
+# force all text files on the working dir to have LF line endings
+# https://git-scm.com/docs/gitattributes#_eol
+* text eol=lf
+
+# PNGs are not text and should not be normalized
+*.png -text

.github/workflows/ci.yml

@@ -0,0 +1,75 @@
+name: ci
+on: [push, pull_request, workflow_dispatch]
+
+jobs:
+  tests:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-20.04, windows-2019]
+        python-version: ["3.13", "3.12", "3.11", "3.10", "3.9", "3.8", "3.7", "3.6"]
+        environment: ['3.8', '3.13', '3.12', '3.11', '3.10', '3.9', '3.7', '3.6', 'interpreter']
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          submodules: recursive
+
+      - uses: actions/setup-python@v5
+        if: ${{ matrix.environment != 'interpreter' }}
+        with:
+          python-version: ${{ matrix.environment }}
+          allow-prereleases: true
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+          allow-prereleases: true
+
+      - name: Install dependencies
+        run: 'pip install .[testing]'
+
+      - name: Run tests
+        run: python -m pytest
+        env:
+          JEDI_TEST_ENVIRONMENT: ${{ matrix.environment }}
+
+  code-quality:
+    runs-on: ubuntu-20.04
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          submodules: recursive
+
+      - name: Install dependencies
+        run: 'pip install .[qa]'
+
+      - name: Run tests
+        run: |
+          python -m flake8 jedi test setup.py
+          python -m mypy jedi sith.py setup.py
+
+  coverage:
+    runs-on: ubuntu-20.04
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          submodules: recursive
+
+      - name: Install dependencies
+        run: 'pip install .[testing] coverage'
+
+      - name: Run tests
+        run: |
+          python -m coverage run --source jedi -m pytest
+          python -m coverage report
+
+      - name: Upload coverage data
+        run: |
+          pip install --quiet codecov coveralls
+          python -m coverage xml
+          python -m coverage report -m
+          bash <(curl -s https://codecov.io/bash) -X gcov -X coveragepy -X search -X fix -X xcode -f coverage.xml

.gitignore

@@ -2,7 +2,6 @@
 *.sw?
 *.pyc
 .ropeproject
-.tox
 .coveralls.yml
 .coverage
 .idea
@@ -13,3 +12,6 @@ jedi.egg-info/
 record.json
 /.cache/
 /.pytest_cache
+/.mypy_cache
+/venv/
+.nvimrc

.gitmodules

@@ -1,3 +1,6 @@
 [submodule "jedi/third_party/typeshed"]
 	path = jedi/third_party/typeshed
 	url = https://github.com/davidhalter/typeshed.git
+[submodule "jedi/third_party/django-stubs"]
+	path = jedi/third_party/django-stubs
+	url = https://github.com/davidhalter/django-stubs

.readthedocs.yml

@@ -0,0 +1,16 @@
+version: 2
+
+python:
+  install:
+    - method: pip
+      path: .
+      extra_requirements:
+        - docs
+
+submodules:
+  include: all
+
+build:
+  os: ubuntu-22.04
+  tools:
+    python: "3.11"

.travis.yml

@@ -1,73 +0,0 @@
-dist: xenial
-language: python
-python:
-  - 2.7
-  - 3.4
-  - 3.5
-  - 3.6
-  - 3.7
-  - 3.8
-
-env:
-  - JEDI_TEST_ENVIRONMENT=27
-  - JEDI_TEST_ENVIRONMENT=34
-  - JEDI_TEST_ENVIRONMENT=35
-  - JEDI_TEST_ENVIRONMENT=36
-  - JEDI_TEST_ENVIRONMENT=37
-  - JEDI_TEST_ENVIRONMENT=38
-  - JEDI_TEST_ENVIRONMENT=interpreter
-
-matrix:
-  include:
-    - python: 3.7
-      env:
-        - TOXENV=cov-py37
-        - JEDI_TEST_ENVIRONMENT=37
-    # For now ignore pypy, there are so many issues that we don't really need
-    # to run it.
-    #- python: pypy
-    # The 3.9 dev build does not seem to be available end 2019.
-    #- python: 3.9-dev
-    #  env:
-    #    - JEDI_TEST_ENVIRONMENT=39
-install:
-    - pip install --quiet tox-travis
-    - sudo apt-get -y install python3-venv
-script:
-    - |
-      # Setup/install Python for $JEDI_TEST_ENVIRONMENT.
-      set -ex
-      test_env_version=${JEDI_TEST_ENVIRONMENT:0:1}.${JEDI_TEST_ENVIRONMENT:1:1}
-      if [ "$TRAVIS_PYTHON_VERSION" != "$test_env_version" ] && [ "$JEDI_TEST_ENVIRONMENT" != "interpreter" ]; then
-        python_bin=python$test_env_version
-        python_path="$(which $python_bin || true)"
-        if [ -z "$python_path" ]; then
-          # Only required for JEDI_TEST_ENVIRONMENT=34.
-          download_name=python-$test_env_version
-          wget https://s3.amazonaws.com/travis-python-archives/binaries/ubuntu/16.04/x86_64/$download_name.tar.bz2
-          sudo tar xjf $download_name.tar.bz2 --directory / opt/python
-          ln -s "/opt/python/${test_env_version}/bin/python" /home/travis/bin/$python_bin
-        elif [ "${python_path#/opt/pyenv/shims}" != "$python_path" ]; then
-          # Activate pyenv version (required with JEDI_TEST_ENVIRONMENT=36).
-          pyenv_bin="$(pyenv whence --path "$python_bin" | head -n1)"
-          ln -s "$pyenv_bin" /home/travis/bin/$python_bin
-        fi
-        $python_bin --version
-        python_ver=$($python_bin -c 'import sys; print("%d%d" % sys.version_info[0:2])')
-        if [ "$JEDI_TEST_ENVIRONMENT" != "$python_ver" ]; then
-          echo "Unexpected Python version for $JEDI_TEST_ENVIRONMENT: $python_ver"
-          set +ex
-          exit 2
-        fi
-      fi
-      set +ex
-    - tox
-after_script:
-    - |
-      if [ $TOXENV == "cov-py37" ]; then
-        pip install --quiet codecov coveralls
-        coverage xml
-        coverage report -m
-        coveralls
-        bash <(curl -s https://codecov.io/bash) -X gcov -X coveragepy -X search -X fix -X xcode -f coverage.xml
-      fi

AUTHORS.txt

@@ -1,59 +1,72 @@
-Main Authors
-============
+Main Authors
+------------
 
-David Halter (@davidhalter) <davidhalter88@gmail.com>
-Takafumi Arakaki (@tkf) <aka.tkf@gmail.com>
+- David Halter (@davidhalter) <davidhalter88@gmail.com>
+- Takafumi Arakaki (@tkf) <aka.tkf@gmail.com>
 
 Code Contributors
-=================
+-----------------
 
-Danilo Bargen (@dbrgn) <mail@dbrgn.ch>
-Laurens Van Houtven (@lvh) <_@lvh.cc>
-Aldo Stracquadanio (@Astrac) <aldo.strac@gmail.com>
-Jean-Louis Fuchs (@ganwell) <ganwell@fangorn.ch>
-tek (@tek)
-Yasha Borevich (@jjay) <j.borevich@gmail.com>
-Aaron Griffin <aaronmgriffin@gmail.com>
-andviro (@andviro)
-Mike Gilbert (@floppym) <floppym@gentoo.org>
-Aaron Meurer (@asmeurer) <asmeurer@gmail.com>
-Lubos Trilety <ltrilety@redhat.com>
-Akinori Hattori (@hattya) <hattya@gmail.com>
-srusskih (@srusskih)
-Steven Silvester (@blink1073)
-Colin Duquesnoy (@ColinDuquesnoy) <colin.duquesnoy@gmail.com>
-Jorgen Schaefer (@jorgenschaefer) <contact@jorgenschaefer.de>
-Fredrik Bergroth (@fbergroth)
-Mathias Fußenegger (@mfussenegger)
-Syohei Yoshida (@syohex) <syohex@gmail.com>
-ppalucky (@ppalucky)
-immerrr (@immerrr) immerrr@gmail.com
-Albertas Agejevas (@alga)
-Savor d'Isavano (@KenetJervet) <newelevenken@163.com>
-Phillip Berndt (@phillipberndt) <phillip.berndt@gmail.com>
-Ian Lee (@IanLee1521) <IanLee1521@gmail.com>
-Farkhad Khatamov (@hatamov) <comsgn@gmail.com>
-Kevin Kelley (@kelleyk) <kelleyk@kelleyk.net>
-Sid Shanker (@squidarth) <sid.p.shanker@gmail.com>
-Reinoud Elhorst (@reinhrst)
-Guido van Rossum (@gvanrossum) <guido@python.org>
-Dmytro Sadovnychyi (@sadovnychyi) <jedi@dmit.ro>
-Cristi Burcă (@scribu)
-bstaint (@bstaint)
-Mathias Rav (@Mortal) <rav@cs.au.dk>
-Daniel Fiterman (@dfit99) <fitermandaniel2@gmail.com>
-Simon Ruggier (@sruggier)
-Élie Gouzien (@ElieGouzien)
-Robin Roth (@robinro)
-Malte Plath (@langsamer)
-Anton Zub (@zabulazza)
-Maksim Novikov (@m-novikov) <mnovikov.work@gmail.com>
-Tobias Rzepka (@TobiasRzepka)
-micbou (@micbou)
-Dima Gerasimov (@karlicoss) <karlicoss@gmail.com>
-Max Woerner Chase (@mwchase) <max.chase@gmail.com>
-Johannes Maria Frank (@jmfrank63) <jmfrank63@gmail.com>
-Shane Steinert-Threlkeld (@shanest) <ssshanest@gmail.com>
-Tim Gates (@timgates42) <tim.gates@iress.com>
+- Danilo Bargen (@dbrgn) <mail@dbrgn.ch>
+- Laurens Van Houtven (@lvh) <_@lvh.cc>
+- Aldo Stracquadanio (@Astrac) <aldo.strac@gmail.com>
+- Jean-Louis Fuchs (@ganwell) <ganwell@fangorn.ch>
+- tek (@tek)
+- Yasha Borevich (@jjay) <j.borevich@gmail.com>
+- Aaron Griffin <aaronmgriffin@gmail.com>
+- andviro (@andviro)
+- Mike Gilbert (@floppym) <floppym@gentoo.org>
+- Aaron Meurer (@asmeurer) <asmeurer@gmail.com>
+- Lubos Trilety <ltrilety@redhat.com>
+- Akinori Hattori (@hattya) <hattya@gmail.com>
+- srusskih (@srusskih)
+- Steven Silvester (@blink1073)
+- Colin Duquesnoy (@ColinDuquesnoy) <colin.duquesnoy@gmail.com>
+- Jorgen Schaefer (@jorgenschaefer) <contact@jorgenschaefer.de>
+- Fredrik Bergroth (@fbergroth)
+- Mathias Fußenegger (@mfussenegger)
+- Syohei Yoshida (@syohex) <syohex@gmail.com>
+- ppalucky (@ppalucky)
+- immerrr (@immerrr) immerrr@gmail.com
+- Albertas Agejevas (@alga)
+- Savor d'Isavano (@KenetJervet) <newelevenken@163.com>
+- Phillip Berndt (@phillipberndt) <phillip.berndt@gmail.com>
+- Ian Lee (@IanLee1521) <IanLee1521@gmail.com>
+- Farkhad Khatamov (@hatamov) <comsgn@gmail.com>
+- Kevin Kelley (@kelleyk) <kelleyk@kelleyk.net>
+- Sid Shanker (@squidarth) <sid.p.shanker@gmail.com>
+- Reinoud Elhorst (@reinhrst)
+- Guido van Rossum (@gvanrossum) <guido@python.org>
+- Dmytro Sadovnychyi (@sadovnychyi) <jedi@dmit.ro>
+- Cristi Burcă (@scribu)
+- bstaint (@bstaint)
+- Mathias Rav (@Mortal) <rav@cs.au.dk>
+- Daniel Fiterman (@dfit99) <fitermandaniel2@gmail.com>
+- Simon Ruggier (@sruggier)
+- Élie Gouzien (@ElieGouzien)
+- Robin Roth (@robinro)
+- Malte Plath (@langsamer)
+- Anton Zub (@zabulazza)
+- Maksim Novikov (@m-novikov) <mnovikov.work@gmail.com>
+- Tobias Rzepka (@TobiasRzepka)
+- micbou (@micbou)
+- Dima Gerasimov (@karlicoss) <karlicoss@gmail.com>
+- Max Woerner Chase (@mwchase) <max.chase@gmail.com>
+- Johannes Maria Frank (@jmfrank63) <jmfrank63@gmail.com>
+- Shane Steinert-Threlkeld (@shanest) <ssshanest@gmail.com>
+- Tim Gates (@timgates42) <tim.gates@iress.com>
+- Lior Goldberg (@goldberglior)
+- Ryan Clary (@mrclary)
+- Max Mäusezahl (@mmaeusezahl) <maxmaeusezahl@googlemail.com>
+- Vladislav Serebrennikov (@endilll)
+- Andrii Kolomoiets (@muffinmad)
+- Leo Ryu (@Leo-Ryu)
+- Joseph Birkner (@josephbirkner)
+- Márcio Mazza (@marciomazza)
+- Martin Vielsmaier (@moser) <martin@vielsmaier.net>
+- TingJia Wu (@WutingjiaX) <wutingjia@bytedance.com>
+- Nguyễn Hồng Quân <ng.hong.quan@gmail.com>
+
+And a few more "anonymous" contributors.
 
 Note: (@user) means a github user name.

CHANGELOG.rst

@@ -3,6 +3,109 @@
 Changelog
 ---------
 
+Unreleased
+++++++++++
+
+- Python 3.13 support
+
+0.19.1 (2023-10-02)
++++++++++++++++++++
+
+- Python 3.12 support (Thanks Peter!)
+
+0.19.0 (2023-07-29)
++++++++++++++++++++
+
+- Python 3.11 support
+- Massive improvements in performance for ``Interpreter`` (e.g. IPython) users.
+  This especially affects ``pandas`` users with large datasets.
+- Add ``jedi.settings.allow_unsafe_interpreter_executions`` to make it easier
+  for IPython users to avoid unsafe executions.
+
+0.18.2 (2022-11-21)
++++++++++++++++++++
+
+- Added dataclass-equivalent for attrs.define
+- Find fixtures from Pytest entrypoints; Examples of pytest plugins installed
+  like this are pytest-django, pytest-sugar and Faker.
+- Fixed Project.search, when a venv was involved, which is why for example
+  `:Pyimport django.db` did not work in some cases in jedi-vim.
+- And many smaller bugfixes
+
+0.18.1 (2021-11-17)
++++++++++++++++++++
+
+- Implict namespaces are now a separate types in ``Name().type``
+- Python 3.10 support
+- Mostly bugfixes
+
+0.18.0 (2020-12-25)
++++++++++++++++++++
+
+- Dropped Python 2 and Python 3.5
+- Using ``pathlib.Path()`` as an output instead of ``str`` in most places:
+  - ``Project.path``
+  - ``Script.path``
+  - ``Definition.module_path``
+  - ``Refactoring.get_renames``
+  - ``Refactoring.get_changed_files``
+- Functions with ``@property`` now return ``property`` instead of ``function``
+  in ``Name().type``
+- Started using annotations
+- Better support for the walrus operator
+- Project attributes are now read accessible
+- Removed all deprecations
+
+This is likely going to be the last minor release before 1.0.
+
+0.17.2 (2020-07-17)
++++++++++++++++++++
+
+- Added an option to pass environment variables to ``Environment``
+- ``Project(...).path`` exists now
+- Support for Python 3.9
+- A few bugfixes
+
+This will be the last release that supports Python 2 and Python 3.5.
+``0.18.0`` will be Python 3.6+.
+
+0.17.1 (2020-06-20)
++++++++++++++++++++
+
+- Django ``Model`` meta class support
+- Django Manager support (completion on Managers/QuerySets)
+- Added Django Stubs to Jedi, thanks to all contributors of the
+  `Django Stubs <https://github.com/typeddjango/django-stubs>`_ project
+- Added ``SyntaxError.get_message``
+- Python 3.9 support
+- Bugfixes (mostly towards Generics)
+
+0.17.0 (2020-04-14)
++++++++++++++++++++
+
+- Added ``Project`` support. This allows a user to specify which folders Jedi
+  should work with.
+- Added support for Refactoring. The following refactorings have been
+  implemented: ``Script.rename``, ``Script.inline``,
+  ``Script.extract_variable`` and ``Script.extract_function``.
+- Added ``Script.get_syntax_errors`` to display syntax errors in the current
+  script.
+- Added code search capabilities both for individual files and projects. The
+  new functions are ``Project.search``, ``Project.complete_search``,
+  ``Script.search`` and ``Script.complete_search``.
+- Added ``Script.help`` to make it easier to display a help window to people.
+  Now returns pydoc information as well for Python keywords/operators.  This
+  means that on the class keyword it will now return the docstring of Python's
+  builtin function ``help('class')``.
+- The API documentation is now way more readable and complete. Check it out
+  under https://jedi.readthedocs.io. A lot of it has been rewritten.
+- Removed Python 3.4 support
+- Many bugfixes
+
+This is likely going to be the last minor version that supports Python 2 and
+Python3.5. Bugfixes will be provided in 0.17.1+. The next minor/major version
+will probably be Jedi 1.0.0.
+
 0.16.0 (2020-01-26)
 +++++++++++++++++++
 
@@ -25,8 +128,8 @@ Changelog
     - ``call_signatures`` deprecated, use ``get_signatures`` instead
     - ``usages`` deprecated, use ``get_references`` instead
     - ``jedi.names`` deprecated, use ``jedi.Script(...).get_names()``
-- ``BaseDefinition.goto_assignments`` renamed to ``BaseDefinition.goto``
-- Add follow_imports to ``Definition.goto``. Now its signature matches
+- ``BaseName.goto_assignments`` renamed to ``BaseName.goto``
+- Add follow_imports to ``Name.goto``. Now its signature matches
   ``Script.goto``.
 - **Python 2 support deprecated**. For this release it is best effort. Python 2
   has reached the end of its life and now it's just about a smooth transition.
@@ -66,13 +169,13 @@ Changelog
 
 New APIs:
 
-- ``Definition.get_signatures() -> List[Signature]``. Signatures are similar to
-  ``CallSignature``. ``Definition.params`` is therefore deprecated.
+- ``Name.get_signatures() -> List[Signature]``. Signatures are similar to
+  ``CallSignature``. ``Name.params`` is therefore deprecated.
 - ``Signature.to_string()`` to format signatures.
-- ``Signature.params -> List[ParamDefinition]``, ParamDefinition has the
+- ``Signature.params -> List[ParamName]``, ParamName has the
   following additional attributes ``infer_default()``, ``infer_annotation()``,
   ``to_string()``, and ``kind``.
-- ``Definition.execute() -> List[Definition]``, makes it possible to infer
+- ``Name.execute() -> List[Name]``, makes it possible to infer
   return values of functions.
 
 
@@ -88,7 +191,7 @@ New APIs:
 - Added ``goto_*(prefer_stubs=True)`` as well as ``goto_*(prefer_stubs=True)``
 - Stubs are used now for type inference
 - Typeshed is used for better type inference
-- Reworked Definition.full_name, should have more correct return values
+- Reworked Name.full_name, should have more correct return values
 
 0.13.3 (2019-02-24)
 +++++++++++++++++++
@@ -168,7 +271,7 @@ New APIs:
 - Actual semantic completions for the complete Python syntax.
 - Basic type inference for ``yield from`` PEP 380.
 - PEP 484 support (most of the important features of it). Thanks Claude! (@reinhrst)
-- Added ``get_line_code`` to ``Definition`` and ``Completion`` objects.
+- Added ``get_line_code`` to ``Name`` and ``Completion`` objects.
 - Completely rewritten the type inference engine.
 - A new and better parser for (fast) parsing diffs of Python code.
 

MANIFEST.in

@@ -6,11 +6,9 @@ include .coveragerc
 include sith.py
 include conftest.py
 include pytest.ini
-include tox.ini
-include requirements.txt
-include jedi/parser/python/grammar*.txt
 recursive-include jedi/third_party *.pyi
 include jedi/third_party/typeshed/LICENSE
+include jedi/third_party/django-stubs/LICENSE.txt
 include jedi/third_party/typeshed/README
 recursive-include test *
 recursive-include docs *

README.rst

@@ -1,117 +1,107 @@
-###################################################################
-Jedi - an awesome autocompletion/static analysis library for Python
-###################################################################
+####################################################################################
+Jedi - an awesome autocompletion, static analysis and refactoring library for Python
+####################################################################################
 
-.. image:: https://img.shields.io/pypi/v/jedi.svg?style=flat
-    :target: https://pypi.python.org/pypi/jedi
-    :alt: PyPI version
+.. image:: http://isitmaintained.com/badge/open/davidhalter/jedi.svg
+    :target: https://github.com/davidhalter/jedi/issues
+    :alt: The percentage of open issues and pull requests
 
-.. image:: https://img.shields.io/pypi/pyversions/jedi.svg
-    :target: https://pypi.python.org/pypi/jedi
-    :alt: Supported Python versions
+.. image:: http://isitmaintained.com/badge/resolution/davidhalter/jedi.svg
+    :target: https://github.com/davidhalter/jedi/issues
+    :alt: The resolution time is the median time an issue or pull request stays open.
 
-.. image:: https://travis-ci.org/davidhalter/jedi.svg?branch=master
-    :target: https://travis-ci.org/davidhalter/jedi
-    :alt: Linux Tests
+.. image:: https://github.com/davidhalter/jedi/workflows/ci/badge.svg?branch=master
+    :target: https://github.com/davidhalter/jedi/actions
+    :alt: Tests
 
-.. image:: https://ci.appveyor.com/api/projects/status/mgva3bbawyma1new/branch/master?svg=true
-    :target: https://ci.appveyor.com/project/davidhalter/jedi/branch/master
-    :alt: Windows Tests
-
-.. image:: https://coveralls.io/repos/davidhalter/jedi/badge.svg?branch=master
-    :target: https://coveralls.io/r/davidhalter/jedi
-    :alt: Coverage status
+.. image:: https://pepy.tech/badge/jedi
+    :target: https://pepy.tech/project/jedi
+    :alt: PyPI Downloads
 
 
-*If you have specific questions, please add an issue or ask on* `Stack Overflow
-<https://stackoverflow.com/questions/tagged/python-jedi>`_ *with the label* ``python-jedi``.
+Jedi is a static analysis tool for Python that is typically used in
+IDEs/editors plugins. Jedi has a focus on autocompletion and goto
+functionality. Other features include refactoring, code search and finding
+references.
 
-
-Jedi is a static analysis tool for Python that can be used in IDEs/editors.
-Jedi has a focus on autocompletion and goto functionality. Jedi is fast and is
-very well tested. It understands Python and stubs on a deep level.
-
-Jedi has support for different goto functions. It's possible to search for
-references and list names in a Python file to get information about them.
-
-Jedi uses a very simple API to connect with IDE's. There's a reference
-implementation as a `VIM-Plugin <https://github.com/davidhalter/jedi-vim>`_,
-which uses Jedi's autocompletion.  We encourage you to use Jedi in your IDEs.
-Autocompletion in your REPL is also possible, IPython uses it natively and for
-the CPython REPL you have to install it.
+Jedi has a simple API to work with. There is a reference implementation as a
+`VIM-Plugin <https://github.com/davidhalter/jedi-vim>`_. Autocompletion in your
+REPL is also possible, IPython uses it natively and for the CPython REPL you
+can install it. Jedi is well tested and bugs should be rare.
 
 Jedi can currently be used with the following editors/projects:
 
 - Vim (jedi-vim_, YouCompleteMe_, deoplete-jedi_, completor.vim_)
+- `Visual Studio Code`_ (via `Python Extension <https://marketplace.visualstudio.com/items?itemName=ms-python.python>`_)
 - Emacs (Jedi.el_, company-mode_, elpy_, anaconda-mode_, ycmd_)
 - Sublime Text (SublimeJEDI_ [ST2 + ST3], anaconda_ [only ST3])
 - TextMate_ (Not sure if it's actually working)
-- Kate_ version 4.13+ supports it natively, you have to enable it, though. [`proof
+- Kate_ version 4.13+ supports it natively, you have to enable it, though.  [`see
   <https://projects.kde.org/projects/kde/applications/kate/repository/show?rev=KDE%2F4.13>`_]
 - Atom_ (autocomplete-python-jedi_)
 - `GNOME Builder`_ (with support for GObject Introspection)
-- `Visual Studio Code`_ (via `Python Extension <https://marketplace.visualstudio.com/items?itemName=ms-python.python>`_)
 - Gedit (gedi_)
 - wdb_ - Web Debugger
-- `Eric IDE`_ (Available as a plugin)
+- `Eric IDE`_
 - `IPython 6.0.0+ <https://ipython.readthedocs.io/en/stable/whatsnew/version6.html>`_
+- `xonsh shell <https://xon.sh/contents.html>`_ has `jedi extension <https://xon.sh/xontribs.html#jedi>`_
 
 and many more!
 
+There are a few language servers that use Jedi:
+
+- `jedi-language-server <https://github.com/pappasam/jedi-language-server>`_
+- `python-language-server <https://github.com/palantir/python-language-server>`_ (currently unmaintained)
+- `python-lsp-server <https://github.com/python-lsp/python-lsp-server>`_ (fork from python-language-server)
+- `anakin-language-server <https://github.com/muffinmad/anakin-language-server>`_
 
 Here are some pictures taken from jedi-vim_:
 
 .. image:: https://github.com/davidhalter/jedi/raw/master/docs/_screenshots/screenshot_complete.png
 
-Completion for almost anything (Ctrl+Space).
+Completion for almost anything:
 
 .. image:: https://github.com/davidhalter/jedi/raw/master/docs/_screenshots/screenshot_function.png
 
-Display of function/class bodies, docstrings.
+Documentation:
 
 .. image:: https://github.com/davidhalter/jedi/raw/master/docs/_screenshots/screenshot_pydoc.png
 
-Pydoc support (Shift+k).
-
-There is also support for goto and renaming.
 
 Get the latest version from `github <https://github.com/davidhalter/jedi>`_
 (master branch should always be kind of stable/working).
 
 Docs are available at `https://jedi.readthedocs.org/en/latest/
-<https://jedi.readthedocs.org/en/latest/>`_. Pull requests with documentation
-enhancements and/or fixes are awesome and most welcome. Jedi uses `semantic
-versioning <https://semver.org/>`_.
+<https://jedi.readthedocs.org/en/latest/>`_. Pull requests with enhancements
+and/or fixes are awesome and most welcome. Jedi uses `semantic versioning
+<https://semver.org/>`_.
 
-If you want to stay up-to-date (News / RFCs), please subscribe to this `github
-thread <https://github.com/davidhalter/jedi/issues/1063>`_.:
+If you want to stay **up-to-date** with releases, please **subscribe** to this
+mailing list: https://groups.google.com/g/jedi-announce. To subscribe you can
+simply send an empty email to ``jedi-announce+subscribe@googlegroups.com``.
 
+Issues & Questions
+==================
 
+You can file issues and questions in the `issue tracker
+<https://github.com/davidhalter/jedi/>`. Alternatively you can also ask on
+`Stack Overflow <https://stackoverflow.com/questions/tagged/python-jedi>`_ with
+the label ``python-jedi``.
 
 Installation
 ============
 
-    pip install jedi
+`Check out the docs <https://jedi.readthedocs.org/en/latest/docs/installation.html>`_.
 
-Note: This just installs the Jedi library, not the editor plugins. For
-information about how to make it work with your editor, refer to the
-corresponding documentation.
+Features and Limitations
+========================
 
-You don't want to use ``pip``? Please refer to the `manual
-<https://jedi.readthedocs.org/en/latest/docs/installation.html>`_.
+Jedi's features are listed here:
+`Features <https://jedi.readthedocs.org/en/latest/docs/features.html>`_.
 
-
-Feature Support and Caveats
-===========================
-
-Jedi really understands your Python code. For a comprehensive list what Jedi
-understands, see: `Features
-<https://jedi.readthedocs.org/en/latest/docs/features.html>`_. A list of
-caveats can be found on the same page.
-
-You can run Jedi on CPython 2.7 or 3.4+ but it should also
-understand/parse code older than those versions. Additionally you should be able
-to use `Virtualenvs <https://jedi.readthedocs.org/en/latest/docs/api.html#environments>`_
+You can run Jedi on Python 3.6+ but it should also
+understand code that is older than those versions. Additionally you should be
+able to use `Virtualenvs <https://jedi.readthedocs.org/en/latest/docs/api.html#environments>`_
 very well.
 
 Tips on how to use Jedi efficiently can be found `here
@@ -120,47 +110,62 @@ Tips on how to use Jedi efficiently can be found `here
 API
 ---
 
-You can find the documentation for the `API here <https://jedi.readthedocs.org/en/latest/docs/api.html>`_.
+You can find a comprehensive documentation for the
+`API here <https://jedi.readthedocs.org/en/latest/docs/api.html>`_.
 
+Autocompletion / Goto / Documentation
+-------------------------------------
 
-Autocompletion / Goto / Pydoc
------------------------------
-
-Please check the API for a good explanation. There are the following commands:
+There are the following commands:
 
 - ``jedi.Script.goto``
+- ``jedi.Script.infer``
+- ``jedi.Script.help``
 - ``jedi.Script.complete``
 - ``jedi.Script.get_references``
+- ``jedi.Script.get_signatures``
+- ``jedi.Script.get_context``
 
-The returned objects are very powerful and really all you might need.
-
+The returned objects are very powerful and are really all you might need.
 
 Autocompletion in your REPL (IPython, etc.)
 -------------------------------------------
 
-Starting with IPython `6.0.0` Jedi is a dependency of IPython. Autocompletion
-in IPython is therefore possible without additional configuration.
+Jedi is a dependency of IPython. Autocompletion in IPython with Jedi is
+therefore possible without additional configuration.
 
-It's possible to have Jedi autocompletion in REPL modes - `example video <https://vimeo.com/122332037>`_.
-This means that in Python you can enable tab completion in a `REPL
+Here is an `example video <https://vimeo.com/122332037>`_ how REPL completion
+can look like.
+For the ``python`` shell you can enable tab completion in a `REPL
 <https://jedi.readthedocs.org/en/latest/docs/usage.html#tab-completion-in-the-python-shell>`_.
 
-
 Static Analysis
-------------------------
+---------------
 
-To do all forms of static analysis, please try to use
-``jedi.Script(...).get_names``. It will return a list of names that you can use
-to infer types and so on.
+For a lot of forms of static analysis, you can try to use
+``jedi.Script(...).get_names``. It will return a list of names that you can
+then filter and work with. There is also a way to list the syntax errors in a
+file: ``jedi.Script.get_syntax_errors``.
 
 
 Refactoring
 -----------
 
-Jedi's parser would support refactoring, but there's no API to use it right
-now.  If you're interested in helping out here, let me know. With the latest
-parser changes, it should be very easy to actually make it work.
+Jedi supports the following refactorings:
 
+- ``jedi.Script.inline``
+- ``jedi.Script.rename``
+- ``jedi.Script.extract_function``
+- ``jedi.Script.extract_variable``
+
+Code Search
+-----------
+
+There is support for module search with ``jedi.Script.search``, and project
+search for ``jedi.Project.search``. The way to search is either by providing a
+name like ``foo`` or by using dotted syntax like ``foo.bar``. Additionally you
+can provide the API type like ``class foo.bar.Bar``. There are also the
+functions ``jedi.Script.complete_search`` and ``jedi.Project.complete_search``.
 
 Development
 ===========
@@ -168,39 +173,26 @@ Development
 There's a pretty good and extensive `development documentation
 <https://jedi.readthedocs.org/en/latest/docs/development.html>`_.
 
-
 Testing
 =======
 
-The test suite depends on ``tox`` and ``pytest``::
+The test suite uses ``pytest``::
 
-    pip install tox pytest
+    pip install pytest
 
-To run the tests for all supported Python versions::
+If you want to test only a specific Python version (e.g. Python 3.8), it is as
+easy as::
 
-    tox
-
-If you want to test only a specific Python version (e.g. Python 2.7), it's as
-easy as ::
-
-    tox -e py27
-
-Tests are also run automatically on `Travis CI
-<https://travis-ci.org/davidhalter/jedi/>`_.
+    python3.8 -m pytest
 
 For more detailed information visit the `testing documentation
 <https://jedi.readthedocs.org/en/latest/docs/testing.html>`_.
 
-
 Acknowledgements
 ================
 
-- Takafumi Arakaki (@tkf) for creating a solid test environment and a lot of
-  other things.
-- Danilo Bargen (@dbrgn) for general housekeeping and being a good friend :).
-- Guido van Rossum (@gvanrossum) for creating the parser generator pgen2
-  (originally used in lib2to3).
-
+Thanks a lot to all the
+`contributors <https://jedi.readthedocs.org/en/latest/docs/acknowledgements.html>`_!
 
 
 .. _jedi-vim: https://github.com/davidhalter/jedi-vim

SECURITY.md

@@ -0,0 +1,9 @@
+# Security Policy
+
+If security issues arise, we will try to fix those as soon as possible.
+
+Due to Jedi's nature, Security Issues will probably be extremely rare, but we will of course treat them seriously.
+
+## Reporting Security Problems
+
+If you need to report a security vulnerability, please send an email to davidhalter88@gmail.com. Typically, I will respond in the next few business days.

appveyor.yml

@@ -1,71 +0,0 @@
-environment:
-  matrix:
-  - TOXENV: py27
-    PYTHON_PATH: C:\Python27
-    JEDI_TEST_ENVIRONMENT: 27
-  - TOXENV: py27
-    PYTHON_PATH: C:\Python27
-    JEDI_TEST_ENVIRONMENT: 34
-  - TOXENV: py27
-    PYTHON_PATH: C:\Python27
-    JEDI_TEST_ENVIRONMENT: 35
-  - TOXENV: py27
-    PYTHON_PATH: C:\Python27
-    JEDI_TEST_ENVIRONMENT: 36
-  - TOXENV: py27
-    PYTHON_PATH: C:\Python27
-    JEDI_TEST_ENVIRONMENT: 37
-
-  - TOXENV: py35
-    PYTHON_PATH: C:\Python35
-    JEDI_TEST_ENVIRONMENT: 27
-  - TOXENV: py35
-    PYTHON_PATH: C:\Python35
-    JEDI_TEST_ENVIRONMENT: 34
-  - TOXENV: py35
-    PYTHON_PATH: C:\Python35
-    JEDI_TEST_ENVIRONMENT: 35
-  - TOXENV: py35
-    PYTHON_PATH: C:\Python35
-    JEDI_TEST_ENVIRONMENT: 36
-  - TOXENV: py35
-    PYTHON_PATH: C:\Python35
-    JEDI_TEST_ENVIRONMENT: 37
-
-  - TOXENV: py36
-    PYTHON_PATH: C:\Python36
-    JEDI_TEST_ENVIRONMENT: 27
-  - TOXENV: py36
-    PYTHON_PATH: C:\Python36
-    JEDI_TEST_ENVIRONMENT: 34
-  - TOXENV: py36
-    PYTHON_PATH: C:\Python36
-    JEDI_TEST_ENVIRONMENT: 35
-  - TOXENV: py36
-    PYTHON_PATH: C:\Python36
-    JEDI_TEST_ENVIRONMENT: 36
-  - TOXENV: py36
-    PYTHON_PATH: C:\Python36
-    JEDI_TEST_ENVIRONMENT: 37
-
-  - TOXENV: py37
-    PYTHON_PATH: C:\Python37
-    JEDI_TEST_ENVIRONMENT: 27
-  - TOXENV: py37
-    PYTHON_PATH: C:\Python37
-    JEDI_TEST_ENVIRONMENT: 34
-  - TOXENV: py37
-    PYTHON_PATH: C:\Python37
-    JEDI_TEST_ENVIRONMENT: 35
-  - TOXENV: py37
-    PYTHON_PATH: C:\Python37
-    JEDI_TEST_ENVIRONMENT: 36
-  - TOXENV: py37
-    PYTHON_PATH: C:\Python37
-    JEDI_TEST_ENVIRONMENT: 37
-install:
-  - git submodule update --init --recursive
-  - set PATH=%PYTHON_PATH%;%PYTHON_PATH%\Scripts;%PATH%
-  - pip install tox
-build_script:
-  - tox

conftest.py

@@ -8,18 +8,16 @@ import pytest
 
 import jedi
 from jedi.api.environment import get_system_environment, InterpreterEnvironment
-from jedi._compatibility import py_version
+from test.helpers import test_dir
 
 collect_ignore = [
     'setup.py',
-    '__main__.py',
+    'jedi/__main__.py',
     'jedi/inference/compiled/subprocess/__main__.py',
     'build/',
     'test/examples',
+    'sith.py',
 ]
-if sys.version_info < (3, 6):
-    # Python 2 not supported syntax
-    collect_ignore.append('test/test_inference/test_mixed.py')
 
 
 # The following hooks (pytest_configure, pytest_unconfigure) are used
@@ -44,7 +42,7 @@ def pytest_addoption(parser):
                      help="Warnings are treated as errors.")
 
     parser.addoption("--env", action='store',
-                     help="Execute the tests in that environment (e.g. 35 for python3.5).")
+                     help="Execute the tests in that environment (e.g. 39 for python3.9).")
     parser.addoption("--interpreter-env", "-I", action='store_true',
                      help="Don't use subprocesses to guarantee having safe "
                           "code execution. Useful for debugging.")
@@ -96,12 +94,15 @@ def clean_jedi_cache(request):
 def environment(request):
     version = request.config.option.env
     if version is None:
-        version = os.environ.get('JEDI_TEST_ENVIRONMENT', str(py_version))
+        v = str(sys.version_info[0]) + str(sys.version_info[1])
+        version = os.environ.get('JEDI_TEST_ENVIRONMENT', v)
 
     if request.config.option.interpreter_env or version == 'interpreter':
         return InterpreterEnvironment()
 
-    return get_system_environment(version[0] + '.' + version[1:])
+    if '.' not in version:
+        version = version[0] + '.' + version[1:]
+    return get_system_environment(version)
 
 
 @pytest.fixture(scope='session')
@@ -109,6 +110,12 @@ def Script(environment):
     return partial(jedi.Script, environment=environment)
 
 
+@pytest.fixture(scope='session')
+def ScriptWithProject(Script):
+    project = jedi.Project(test_dir)
+    return partial(jedi.Script, project=project)
+
+
 @pytest.fixture(scope='session')
 def get_names(Script):
     return lambda code, **kwargs: Script(code).get_names(**kwargs)
@@ -126,17 +133,21 @@ def goto_or_help(request, Script):
 
 @pytest.fixture(scope='session', params=['goto', 'help', 'infer'])
 def goto_or_help_or_infer(request, Script):
+    def do(code, *args, **kwargs):
+        return getattr(Script(code), request.param)(*args, **kwargs)
+
+    do.type = request.param
+    return do
+
+
+@pytest.fixture(scope='session', params=['goto', 'complete', 'help'])
+def goto_or_complete(request, Script):
     return lambda code, *args, **kwargs: getattr(Script(code), request.param)(*args, **kwargs)
 
 
 @pytest.fixture(scope='session')
-def has_typing(environment):
-    if environment.version_info >= (3, 5, 0):
-        # This if is just needed to avoid that tests ever skip way more than
-        # they should for all Python versions.
-        return True
-
-    script = jedi.Script('import typing', environment=environment)
+def has_django(environment):
+    script = jedi.Script('import django', environment=environment)
     return bool(script.infer())
 
 
@@ -145,14 +156,6 @@ def jedi_path():
     return os.path.dirname(__file__)
 
 
-@pytest.fixture()
-def skip_python2(environment):
-    if environment.version_info.major == 2:
-        # This if is just needed to avoid that tests ever skip way more than
-        # they should for all Python versions.
-        pytest.skip()
-
-
 @pytest.fixture()
 def skip_pre_python38(environment):
     if environment.version_info < (3, 8):
@@ -167,19 +170,3 @@ def skip_pre_python37(environment):
         # This if is just needed to avoid that tests ever skip way more than
         # they should for all Python versions.
         pytest.skip()
-
-
-@pytest.fixture()
-def skip_pre_python35(environment):
-    if environment.version_info < (3, 5):
-        # This if is just needed to avoid that tests ever skip way more than
-        # they should for all Python versions.
-        pytest.skip()
-
-
-@pytest.fixture()
-def skip_pre_python36(environment):
-    if environment.version_info < (3, 6):
-        # This if is just needed to avoid that tests ever skip way more than
-        # they should for all Python versions.
-        pytest.skip()

deploy-master.sh

@@ -24,10 +24,10 @@ git checkout $BRANCH
 git submodule update --init
 
 # Test first.
-tox
+pytest
 
 # Create tag
-tag=v$(python -c "import $PROJECT_NAME; print($PROJECT_NAME.__version__)")
+tag=v$(python3 -c "import $PROJECT_NAME; print($PROJECT_NAME.__version__)")
 
 master_ref=$(git show-ref -s heads/$BRANCH)
 tag_ref=$(git show-ref -s $tag || true)
@@ -44,7 +44,7 @@ fi
 # Package and upload to PyPI
 #rm -rf dist/ - Not needed anymore, because the folder is never reused.
 echo `pwd`
-python setup.py sdist bdist_wheel
+python3 setup.py sdist bdist_wheel
 # Maybe do a pip install twine before.
 twine upload dist/*
 

docs/_static/custom_style.css

@@ -0,0 +1,9 @@
+div.version {
+    color: black !important;
+    margin-top: -1.2em !important;
+    margin-bottom: .6em !important;
+}
+
+div.wy-side-nav-search {
+    padding-top: 0 !important;
+}

docs/_themes/flask/LICENSE

@@ -1,37 +0,0 @@
-Copyright (c) 2010 by Armin Ronacher.
-
-Some rights reserved.
-
-Redistribution and use in source and binary forms of the theme, with or
-without modification, are permitted provided that the following conditions
-are met:
-
-* Redistributions of source code must retain the above copyright
-  notice, this list of conditions and the following disclaimer.
-
-* Redistributions in binary form must reproduce the above
-  copyright notice, this list of conditions and the following
-  disclaimer in the documentation and/or other materials provided
-  with the distribution.
-
-* The names of the contributors may not be used to endorse or
-  promote products derived from this software without specific
-  prior written permission.
-
-We kindly ask you to only use these themes in an unmodified manner just
-for Flask and Flask-related products, not for unrelated projects.  If you
-like the visual style and want to use it for your own projects, please
-consider making some larger changes to the themes (such as changing
-font faces, sizes, colors or margins).
-
-THIS THEME IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
-LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-ARISING IN ANY WAY OUT OF THE USE OF THIS THEME, EVEN IF ADVISED OF THE
-POSSIBILITY OF SUCH DAMAGE.

docs/_themes/flask/layout.html

@@ -1,27 +0,0 @@
-{%- extends "basic/layout.html" %}
-{%- block extrahead %}
-  {{ super() }}
-  {% if theme_touch_icon %}
-  <link rel="apple-touch-icon" href="{{ pathto('_static/' ~ theme_touch_icon, 1) }}" />
-  {% endif %}
-  <link media="only screen and (max-device-width: 480px)" href="{{
-    pathto('_static/small_flask.css', 1) }}" type= "text/css" rel="stylesheet" />
-  <a href="https://github.com/davidhalter/jedi">
-    <img style="position: absolute; top: 0; right: 0; border: 0;" src="https://s3.amazonaws.com/github/ribbons/forkme_right_red_aa0000.png" alt="Fork me on GitHub">
-  </a>
-{% endblock %}
-{%- block relbar2 %}{% endblock %}
-{% block header %}
-  {{ super() }}
-  {% if pagename == 'index' %}
-  <div class=indexwrapper>
-  {% endif %}
-{% endblock %}
-{%- block footer %}
-  <div class="footer">
-    Created using <a href="http://sphinx.pocoo.org/">Sphinx</a>.
-  </div>
-  {% if pagename == 'index' %}
-  </div>
-  {% endif %}
-{%- endblock %}

docs/_themes/flask/relations.html

@@ -1,19 +0,0 @@
-<h3>Related Topics</h3>
-<ul>
-  <li><a href="{{ pathto(master_doc) }}">Documentation overview</a><ul>
-  {%- for parent in parents %}
-  <li><a href="{{ parent.link|e }}">{{ parent.title }}</a><ul>
-  {%- endfor %}
-    {%- if prev %}
-      <li>Previous: <a href="{{ prev.link|e }}" title="{{ _('previous chapter')
-        }}">{{ prev.title }}</a></li>
-    {%- endif %}
-    {%- if next %}
-      <li>Next: <a href="{{ next.link|e }}" title="{{ _('next chapter')
-        }}">{{ next.title }}</a></li>
-    {%- endif %}
-  {%- for parent in parents %}
-  </ul></li>
-  {%- endfor %}
-  </ul></li>
-</ul>

docs/_themes/flask/static/flasky.css_t

@@ -1,394 +0,0 @@
-/*
- * flasky.css_t
- * ~~~~~~~~~~~~
- *
- * :copyright: Copyright 2010 by Armin Ronacher.
- * :license: Flask Design License, see LICENSE for details.
- */
-
-{% set page_width = '940px' %}
-{% set sidebar_width = '220px' %}
- 
-@import url("basic.css");
- 
-/* -- page layout ----------------------------------------------------------- */
- 
-body {
-    font-family: 'Georgia', serif;
-    font-size: 17px;
-    background-color: white;
-    color: #000;
-    margin: 0;
-    padding: 0;
-}
-
-div.document {
-    width: {{ page_width }};
-    margin: 30px auto 0 auto;
-}
-
-div.documentwrapper {
-    float: left;
-    width: 100%;
-}
-
-div.bodywrapper {
-    margin: 0 0 0 {{ sidebar_width }};
-}
-
-div.sphinxsidebar {
-    width: {{ sidebar_width }};
-}
-
-hr {
-    border: 1px solid #B1B4B6;
-}
- 
-div.body {
-    background-color: #ffffff;
-    color: #3E4349;
-    padding: 0 30px 0 30px;
-}
-
-img.floatingflask {
-    padding: 0 0 10px 10px;
-    float: right;
-}
- 
-div.footer {
-    width: {{ page_width }};
-    margin: 20px auto 30px auto;
-    font-size: 14px;
-    color: #888;
-    text-align: right;
-}
-
-div.footer a {
-    color: #888;
-}
-
-div.related {
-    display: none;
-}
- 
-div.sphinxsidebar a {
-    color: #444;
-    text-decoration: none;
-    border-bottom: 1px dotted #999;
-}
-
-div.sphinxsidebar a:hover {
-    border-bottom: 1px solid #999;
-}
- 
-div.sphinxsidebar {
-    font-size: 14px;
-    line-height: 1.5;
-}
-
-div.sphinxsidebarwrapper {
-    padding: 18px 10px;
-}
-
-div.sphinxsidebarwrapper p.logo {
-    padding: 0 0 20px 0;
-    margin: 0;
-    text-align: center;
-}
- 
-div.sphinxsidebar h3,
-div.sphinxsidebar h4 {
-    font-family: 'Garamond', 'Georgia', serif;
-    color: #444;
-    font-size: 24px;
-    font-weight: normal;
-    margin: 0 0 5px 0;
-    padding: 0;
-}
-
-div.sphinxsidebar h4 {
-    font-size: 20px;
-}
- 
-div.sphinxsidebar h3 a {
-    color: #444;
-}
-
-div.sphinxsidebar p.logo a,
-div.sphinxsidebar h3 a,
-div.sphinxsidebar p.logo a:hover,
-div.sphinxsidebar h3 a:hover {
-    border: none;
-}
- 
-div.sphinxsidebar p {
-    color: #555;
-    margin: 10px 0;
-}
-
-div.sphinxsidebar ul {
-    margin: 10px 0;
-    padding: 0;
-    color: #000;
-}
- 
-div.sphinxsidebar input {
-    border: 1px solid #ccc;
-    font-family: 'Georgia', serif;
-    font-size: 1em;
-}
- 
-/* -- body styles ----------------------------------------------------------- */
- 
-a {
-    color: #004B6B;
-    text-decoration: underline;
-}
- 
-a:hover {
-    color: #6D4100;
-    text-decoration: underline;
-}
- 
-div.body h1,
-div.body h2,
-div.body h3,
-div.body h4,
-div.body h5,
-div.body h6 {
-    font-family: 'Garamond', 'Georgia', serif;
-    font-weight: normal;
-    margin: 30px 0px 10px 0px;
-    padding: 0;
-}
-
-{% if theme_index_logo %}
-div.indexwrapper h1 {
-    text-indent: -999999px;
-    background: url({{ theme_index_logo }}) no-repeat center center;
-    height: {{ theme_index_logo_height }};
-}
-{% endif %}
- 
-div.body h1 { margin-top: 0; padding-top: 0; font-size: 240%; }
-div.body h2 { font-size: 180%; }
-div.body h3 { font-size: 150%; }
-div.body h4 { font-size: 130%; }
-div.body h5 { font-size: 100%; }
-div.body h6 { font-size: 100%; }
- 
-a.headerlink {
-    color: #ddd;
-    padding: 0 4px;
-    text-decoration: none;
-}
- 
-a.headerlink:hover {
-    color: #444;
-}
- 
-div.body p, div.body dd, div.body li {
-    line-height: 1.4em;
-}
-
-div.admonition {
-    background: #fafafa;
-    margin: 20px -30px;
-    padding: 10px 30px;
-    border-top: 1px solid #ccc;
-    border-bottom: 1px solid #ccc;
-}
-
-div.admonition tt.xref, div.admonition a tt {
-    border-bottom: 1px solid #fafafa;
-}
-
-dd div.admonition {
-    margin-left: -60px;
-    padding-left: 60px;
-}
-
-div.admonition p.admonition-title {
-    font-family: 'Garamond', 'Georgia', serif;
-    font-weight: normal;
-    font-size: 24px;
-    margin: 0 0 10px 0;
-    padding: 0;
-    line-height: 1;
-}
-
-div.admonition p.last {
-    margin-bottom: 0;
-}
-
-div.highlight {
-    background-color: white;
-}
-
-dt:target, .highlight {
-    background: #FAF3E8;
-}
-
-div.note {
-    background-color: #eee;
-    border: 1px solid #ccc;
-}
- 
-div.seealso {
-    background-color: #ffc;
-    border: 1px solid #ff6;
-}
- 
-div.topic {
-    background-color: #eee;
-}
- 
-p.admonition-title {
-    display: inline;
-}
- 
-p.admonition-title:after {
-    content: ":";
-}
-
-pre, tt {
-    font-family: 'Consolas', 'Menlo', 'Deja Vu Sans Mono', 'Bitstream Vera Sans Mono', monospace;
-    font-size: 0.9em;
-}
-
-img.screenshot {
-}
-
-tt.descname, tt.descclassname {
-    font-size: 0.95em;
-}
-
-tt.descname {
-    padding-right: 0.08em;
-}
-
-img.screenshot {
-    -moz-box-shadow: 2px 2px 4px #eee;
-    -webkit-box-shadow: 2px 2px 4px #eee;
-    box-shadow: 2px 2px 4px #eee;
-}
-
-table.docutils {
-    border: 1px solid #888;
-    -moz-box-shadow: 2px 2px 4px #eee;
-    -webkit-box-shadow: 2px 2px 4px #eee;
-    box-shadow: 2px 2px 4px #eee;
-}
-
-table.docutils td, table.docutils th {
-    border: 1px solid #888;
-    padding: 0.25em 0.7em;
-}
-
-table.field-list, table.footnote {
-    border: none;
-    -moz-box-shadow: none;
-    -webkit-box-shadow: none;
-    box-shadow: none;
-}
-
-table.footnote {
-    margin: 15px 0;
-    width: 100%;
-    border: 1px solid #eee;
-    background: #fdfdfd;
-    font-size: 0.9em;
-}
-
-table.footnote + table.footnote {
-    margin-top: -15px;
-    border-top: none;
-}
-
-table.field-list th {
-    padding: 0 0.8em 0 0;
-}
-
-table.field-list td {
-    padding: 0;
-}
-
-table.footnote td.label {
-    width: 0px;
-    padding: 0.3em 0 0.3em 0.5em;
-}
-
-table.footnote td {
-    padding: 0.3em 0.5em;
-}
-
-dl {
-    margin: 0;
-    padding: 0;
-}
-
-dl dd {
-    margin-left: 30px;
-}
-
-blockquote {
-    margin: 0 0 0 30px;
-    padding: 0;
-}
-
-ul, ol {
-    margin: 10px 0 10px 30px;
-    padding: 0;
-}
- 
-pre {
-    background: #eee;
-    padding: 7px 30px;
-    margin: 15px -30px;
-    line-height: 1.3em;
-}
-
-dl pre, blockquote pre, li pre {
-    margin-left: -60px;
-    padding-left: 60px;
-}
-
-dl dl pre {
-    margin-left: -90px;
-    padding-left: 90px;
-}
- 
-tt {
-    background-color: #ecf0f3;
-    color: #222;
-    /* padding: 1px 2px; */
-}
-
-tt.xref, a tt {
-    background-color: #FBFBFB;
-    border-bottom: 1px solid white;
-}
-
-a.reference {
-    text-decoration: none;
-    border-bottom: 1px dotted #004B6B;
-}
-
-a.reference:hover {
-    border-bottom: 1px solid #6D4100;
-}
-
-a.footnote-reference {
-    text-decoration: none;
-    font-size: 0.7em;
-    vertical-align: top;
-    border-bottom: 1px dotted #004B6B;
-}
-
-a.footnote-reference:hover {
-    border-bottom: 1px solid #6D4100;
-}
-
-a:hover tt {
-    background: #EEE;
-}

docs/_themes/flask/static/small_flask.css

@@ -1,70 +0,0 @@
-/*
- * small_flask.css_t
- * ~~~~~~~~~~~~~~~~~
- *
- * :copyright: Copyright 2010 by Armin Ronacher.
- * :license: Flask Design License, see LICENSE for details.
- */
-
-body {
-    margin: 0;
-    padding: 20px 30px;
-}
-
-div.documentwrapper {
-    float: none;
-    background: white;
-}
-
-div.sphinxsidebar {
-    display: block;
-    float: none;
-    width: 102.5%;
-    margin: 50px -30px -20px -30px;
-    padding: 10px 20px;
-    background: #333;
-    color: white;
-}
-
-div.sphinxsidebar h3, div.sphinxsidebar h4, div.sphinxsidebar p,
-div.sphinxsidebar h3 a {
-    color: white;
-}
-
-div.sphinxsidebar a {
-    color: #aaa;
-}
-
-div.sphinxsidebar p.logo {
-    display: none;
-}
-
-div.document {
-    width: 100%;
-    margin: 0;
-}
-
-div.related {
-    display: block;
-    margin: 0;
-    padding: 10px 0 20px 0;
-}
-
-div.related ul,
-div.related ul li {
-    margin: 0;
-    padding: 0;
-}
-
-div.footer {
-    display: none;
-}
-
-div.bodywrapper {
-    margin: 0;
-}
-
-div.body {
-    min-height: 0;
-    padding: 0;
-}

docs/_themes/flask/theme.conf

@@ -1,9 +0,0 @@
-[theme]
-inherit = basic
-stylesheet = flasky.css
-pygments_style = flask_theme_support.FlaskyStyle
-
-[options]
-index_logo = 
-index_logo_height = 120px
-touch_icon = 

docs/_themes/flask_theme_support.py

@@ -1,125 +0,0 @@
-"""
-Copyright (c) 2010 by Armin Ronacher.
-
-Some rights reserved.
-
-Redistribution and use in source and binary forms of the theme, with or
-without modification, are permitted provided that the following conditions
-are met:
-
-* Redistributions of source code must retain the above copyright
-  notice, this list of conditions and the following disclaimer.
-
-* Redistributions in binary form must reproduce the above
-  copyright notice, this list of conditions and the following
-  disclaimer in the documentation and/or other materials provided
-  with the distribution.
-
-* The names of the contributors may not be used to endorse or
-  promote products derived from this software without specific
-  prior written permission.
-
-We kindly ask you to only use these themes in an unmodified manner just
-for Flask and Flask-related products, not for unrelated projects.  If you
-like the visual style and want to use it for your own projects, please
-consider making some larger changes to the themes (such as changing
-font faces, sizes, colors or margins).
-
-THIS THEME IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
-LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-ARISING IN ANY WAY OUT OF THE USE OF THIS THEME, EVEN IF ADVISED OF THE
-POSSIBILITY OF SUCH DAMAGE.
-"""
-# flasky extensions.  flasky pygments style based on tango style
-from pygments.style import Style
-from pygments.token import Keyword, Name, Comment, String, Error, \
-     Number, Operator, Generic, Whitespace, Punctuation, Other, Literal
-
-
-class FlaskyStyle(Style):
-    background_color = "#f8f8f8"
-    default_style = ""
-
-    styles = {
-        # No corresponding class for the following:
-        #Text:                     "", # class:  ''
-        Whitespace:                "underline #f8f8f8",      # class: 'w'
-        Error:                     "#a40000 border:#ef2929", # class: 'err'
-        Other:                     "#000000",                # class 'x'
-
-        Comment:                   "italic #8f5902", # class: 'c'
-        Comment.Preproc:           "noitalic",       # class: 'cp'
-
-        Keyword:                   "bold #004461",   # class: 'k'
-        Keyword.Constant:          "bold #004461",   # class: 'kc'
-        Keyword.Declaration:       "bold #004461",   # class: 'kd'
-        Keyword.Namespace:         "bold #004461",   # class: 'kn'
-        Keyword.Pseudo:            "bold #004461",   # class: 'kp'
-        Keyword.Reserved:          "bold #004461",   # class: 'kr'
-        Keyword.Type:              "bold #004461",   # class: 'kt'
-
-        Operator:                  "#582800",   # class: 'o'
-        Operator.Word:             "bold #004461",   # class: 'ow' - like keywords
-
-        Punctuation:               "bold #000000",   # class: 'p'
-
-        # because special names such as Name.Class, Name.Function, etc.
-        # are not recognized as such later in the parsing, we choose them
-        # to look the same as ordinary variables.
-        Name:                      "#000000",        # class: 'n'
-        Name.Attribute:            "#c4a000",        # class: 'na' - to be revised
-        Name.Builtin:              "#004461",        # class: 'nb'
-        Name.Builtin.Pseudo:       "#3465a4",        # class: 'bp'
-        Name.Class:                "#000000",        # class: 'nc' - to be revised
-        Name.Constant:             "#000000",        # class: 'no' - to be revised
-        Name.Decorator:            "#888",           # class: 'nd' - to be revised
-        Name.Entity:               "#ce5c00",        # class: 'ni'
-        Name.Exception:            "bold #cc0000",   # class: 'ne'
-        Name.Function:             "#000000",        # class: 'nf'
-        Name.Property:             "#000000",        # class: 'py'
-        Name.Label:                "#f57900",        # class: 'nl'
-        Name.Namespace:            "#000000",        # class: 'nn' - to be revised
-        Name.Other:                "#000000",        # class: 'nx'
-        Name.Tag:                  "bold #004461",   # class: 'nt' - like a keyword
-        Name.Variable:             "#000000",        # class: 'nv' - to be revised
-        Name.Variable.Class:       "#000000",        # class: 'vc' - to be revised
-        Name.Variable.Global:      "#000000",        # class: 'vg' - to be revised
-        Name.Variable.Instance:    "#000000",        # class: 'vi' - to be revised
-
-        Number:                    "#990000",        # class: 'm'
-
-        Literal:                   "#000000",        # class: 'l'
-        Literal.Date:              "#000000",        # class: 'ld'
-
-        String:                    "#4e9a06",        # class: 's'
-        String.Backtick:           "#4e9a06",        # class: 'sb'
-        String.Char:               "#4e9a06",        # class: 'sc'
-        String.Doc:                "italic #8f5902", # class: 'sd' - like a comment
-        String.Double:             "#4e9a06",        # class: 's2'
-        String.Escape:             "#4e9a06",        # class: 'se'
-        String.Heredoc:            "#4e9a06",        # class: 'sh'
-        String.Interpol:           "#4e9a06",        # class: 'si'
-        String.Other:              "#4e9a06",        # class: 'sx'
-        String.Regex:              "#4e9a06",        # class: 'sr'
-        String.Single:             "#4e9a06",        # class: 's1'
-        String.Symbol:             "#4e9a06",        # class: 'ss'
-
-        Generic:                   "#000000",        # class: 'g'
-        Generic.Deleted:           "#a40000",        # class: 'gd'
-        Generic.Emph:              "italic #000000", # class: 'ge'
-        Generic.Error:             "#ef2929",        # class: 'gr'
-        Generic.Heading:           "bold #000080",   # class: 'gh'
-        Generic.Inserted:          "#00A000",        # class: 'gi'
-        Generic.Output:            "#888",           # class: 'go'
-        Generic.Prompt:            "#745334",        # class: 'gp'
-        Generic.Strong:            "bold #000000",   # class: 'gs'
-        Generic.Subheading:        "bold #800080",   # class: 'gu'
-        Generic.Traceback:         "bold #a40000",   # class: 'gt'
-    }

docs/conf.py

@@ -1,5 +1,3 @@
-# -*- coding: utf-8 -*-
-#
 # Jedi documentation build configuration file, created by
 # sphinx-quickstart on Wed Dec 26 00:11:34 2012.
 #
@@ -13,13 +11,11 @@
 
 import sys
 import os
-import datetime
 
 # If extensions (or modules to document with autodoc) are in another directory,
 # add these directories to sys.path here. If the directory is relative to the
 # documentation root, use os.path.abspath to make it absolute, like shown here.
 sys.path.insert(0, os.path.abspath('..'))
-sys.path.append(os.path.abspath('_themes'))
 
 # -- General configuration -----------------------------------------------------
 
@@ -29,7 +25,8 @@ sys.path.append(os.path.abspath('_themes'))
 # Add any Sphinx extension module names here, as strings. They can be extensions
 # coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
 extensions = ['sphinx.ext.autodoc', 'sphinx.ext.viewcode', 'sphinx.ext.todo',
-              'sphinx.ext.intersphinx', 'sphinx.ext.inheritance_diagram']
+              'sphinx.ext.intersphinx', 'sphinx.ext.inheritance_diagram',
+              'sphinx_rtd_theme', 'sphinx.ext.autosummary']
 
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ['_templates']
@@ -44,8 +41,8 @@ source_encoding = 'utf-8'
 master_doc = 'index'
 
 # General information about the project.
-project = u'Jedi'
-copyright = u'jedi contributors'
+project = 'Jedi'
+copyright = 'jedi contributors'
 
 import jedi
 from jedi.utils import version_info
@@ -54,8 +51,8 @@ from jedi.utils import version_info
 # |version| and |release|, also used in various other places throughout the
 # built documents.
 #
-# The short X.Y version.
-version = '.'.join(str(x) for x in version_info()[:2])
+# The short X.Y.Z version.
+version = '.'.join(str(x) for x in version_info()[:3])
 # The full version, including alpha/beta/rc tags.
 release = jedi.__version__
 
@@ -98,12 +95,15 @@ pygments_style = 'sphinx'
 
 # The theme to use for HTML and HTML Help pages.  See the documentation for
 # a list of builtin themes.
-html_theme = 'flask'
+html_theme = 'sphinx_rtd_theme'
 
 # Theme options are theme-specific and customize the look and feel of a theme
 # further.  For a list of options available for each theme, see the
 # documentation.
-#html_theme_options = {}
+html_theme_options = {
+    'logo_only': True,
+    'style_nav_header_background': 'white',
+}
 
 # Add any paths that contain custom themes here, relative to this directory.
 html_theme_path = ['_themes']
@@ -117,7 +117,7 @@ html_theme_path = ['_themes']
 
 # The name of an image file (relative to this directory) to place at the top
 # of the sidebar.
-#html_logo = None
+html_logo = '_static/logo.png'
 
 # The name of an image file (within the static path) to use as favicon of the
 # docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
@@ -129,6 +129,8 @@ html_theme_path = ['_themes']
 # so a file named "default.css" will overwrite the builtin "default.css".
 html_static_path = ['_static']
 
+html_css_files = ['custom_style.css']
+
 # If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
 # using the given strftime format.
 #html_last_updated_fmt = '%b %d, %Y'
@@ -145,7 +147,7 @@ html_sidebars = {
         #'relations.html',
         'ghbuttons.html',
         #'sourcelink.html',
-        #'searchbox.html'
+        'searchbox.html'
     ]
 }
 
@@ -163,13 +165,13 @@ html_sidebars = {
 #html_split_index = False
 
 # If true, links to the reST sources are added to the pages.
-#html_show_sourcelink = True
+html_show_sourcelink = False
 
 # If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
-#html_show_sphinx = True
+html_show_sphinx = False
 
 # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
-#html_show_copyright = True
+html_show_copyright = False
 
 # If true, an OpenSearch description file will be output, and all pages will
 # contain a <link> tag referring to it.  The value of this option must be the
@@ -201,8 +203,8 @@ latex_elements = {
 # Grouping the document tree into LaTeX files. List of tuples
 # (source start file, target name, title, author, documentclass [howto/manual]).
 latex_documents = [
-    ('index', 'Jedi.tex', u'Jedi Documentation',
-     u'Jedi contributors', 'manual'),
+    ('index', 'Jedi.tex', 'Jedi Documentation',
+     'Jedi contributors', 'manual'),
 ]
 
 # The name of an image file (relative to this directory) to place at the top of
@@ -231,8 +233,8 @@ latex_documents = [
 # One entry per manual page. List of tuples
 # (source start file, name, description, authors, manual section).
 man_pages = [
-    ('index', 'jedi', u'Jedi Documentation',
-     [u'Jedi contributors'], 1)
+    ('index', 'jedi', 'Jedi Documentation',
+     ['Jedi contributors'], 1)
 ]
 
 # If true, show URL addresses after external links.
@@ -245,8 +247,8 @@ man_pages = [
 # (source start file, target name, title, author,
 #  dir menu entry, description, category)
 texinfo_documents = [
-    ('index', 'Jedi', u'Jedi Documentation',
-     u'Jedi contributors', 'Jedi', 'Awesome Python autocompletion library.',
+    ('index', 'Jedi', 'Jedi Documentation',
+     'Jedi contributors', 'Jedi', 'Awesome Python autocompletion library.',
      'Miscellaneous'),
 ]
 
@@ -274,7 +276,8 @@ autodoc_default_flags = []
 # -- Options for intersphinx module --------------------------------------------
 
 intersphinx_mapping = {
-    'https://docs.python.org/': None,
+    'python': ('https://docs.python.org/', None),
+    'parso': ('https://parso.readthedocs.io/en/latest/', None),
 }
 
 

docs/docs/acknowledgements.rst

@@ -0,0 +1,66 @@
+.. include global.rst
+
+History & Acknowledgements
+==========================
+
+Acknowledgements
+----------------
+
+- Dave Halter for creating and maintaining Jedi & Parso.
+- Takafumi Arakaki (@tkf) for creating a solid test environment and a lot of
+  other things.
+- Danilo Bargen (@dbrgn) for general housekeeping and being a good friend :).
+- Guido van Rossum (@gvanrossum) for creating the parser generator pgen2
+  (originally used in lib2to3).
+- Thanks to all the :ref:`contributors <contributors>`.
+
+A Little Bit of History
+-----------------------
+
+Written by Dave.
+
+The Star Wars Jedi are awesome. My Jedi software tries to imitate a little bit
+of the precognition the Jedi have. There's even an awesome `scene
+<https://youtu.be/yHRJLIf7wMU>`_ of Monty Python Jedis :-).
+
+But actually the name has not much to do with Star Wars. It's part of my
+second name Jedidjah.
+
+I actually started Jedi back in 2012, because there were no good solutions
+available for VIM. Most auto-completion solutions just did not work well. The
+only good solution was PyCharm. But I liked my good old VIM very much. There
+was also a solution called Rope that did not work at all for me. So I decided
+to write my own version of a completion engine.
+
+The first idea was to execute non-dangerous code. But I soon realized, that
+this would not work. So I started to build a static analysis tool.
+The biggest problem that I had at the time was that I did not know a thing
+about parsers. I did not even know the word static analysis. It turns
+out they are the foundation of a good static analysis tool. I of course did not
+know that and tried to write my own poor version of a parser that I ended up
+throwing away two years later.
+
+Because of my lack of knowledge, everything after 2012 and before 2020 was
+basically refactoring. I rewrote the core parts of Jedi probably like 5-10
+times. The last big rewrite (that I did twice) was the inclusion of
+gradual typing and stubs.
+
+I learned during that time that it is crucial to have a good understanding of
+your problem. Otherwise you just end up doing it again. I only wrote features
+in the beginning and in the end. Everything else was bugfixing and refactoring.
+However now I am really happy with the result. It works well, bugfixes can be
+quick and is pretty much feature complete.
+
+--------
+
+I will leave you with a small anecdote that happened in 2012, if I remember
+correctly. After I explained Guido van Rossum, how some parts of my
+auto-completion work, he said:
+
+    *"Oh, that worries me..."*
+
+Now that it is finished, I hope he likes it :-).
+
+.. _contributors:
+
+.. include:: ../../AUTHORS.txt

docs/docs/api-classes.rst

@@ -5,6 +5,49 @@
 API Return Classes
 ------------------
 
-.. automodule:: jedi.api.classes
+Abstract Base Class
+~~~~~~~~~~~~~~~~~~~
+.. autoclass:: jedi.api.classes.BaseName
     :members:
-    :undoc-members:
+    :show-inheritance:
+
+Name
+~~~~
+.. autoclass:: jedi.api.classes.Name
+    :members:
+    :show-inheritance:
+
+Completion
+~~~~~~~~~~
+.. autoclass:: jedi.api.classes.Completion
+    :members:
+    :show-inheritance:
+
+BaseSignature
+~~~~~~~~~~~~~
+.. autoclass:: jedi.api.classes.BaseSignature
+    :members:
+    :show-inheritance:
+
+Signature
+~~~~~~~~~
+.. autoclass:: jedi.api.classes.Signature
+    :members:
+    :show-inheritance:
+
+ParamName
+~~~~~~~~~
+.. autoclass:: jedi.api.classes.ParamName
+    :members:
+    :show-inheritance:
+
+Refactoring
+~~~~~~~~~~~
+
+.. autoclass:: jedi.api.refactoring.Refactoring
+    :members:
+    :show-inheritance:
+
+.. autoclass:: jedi.api.errors.SyntaxError
+    :members:
+    :show-inheritance:

docs/docs/api.rst

@@ -3,56 +3,74 @@
 API Overview
 ============
 
-.. currentmodule:: jedi
-
-Note: This documentation is for Plugin developers, who want to improve their
-editors/IDE autocompletion 
-
-If you want to use |jedi|, you first need to ``import jedi``.  You then have
-direct access to the :class:`.Script`. You can then call the functions
-documented here. These functions return :ref:`API classes
-<api-classes>`.
-
-
-Deprecations
-------------
-
-The deprecation process is as follows:
-
-1. A deprecation is announced in the next major/minor release.
-2. We wait either at least a year & at least two minor releases until we remove
-   the deprecated functionality.
-
-
-API Documentation
------------------
-
-The API consists of a few different parts:
-
-- The main starting points for complete/goto: :class:`.Script` and :class:`.Interpreter`
-- Helpful functions: :func:`.preload_module` and :func:`.set_debug_function`
-- :ref:`API Result Classes <api-classes>`
-- :ref:`Python Versions/Virtualenv Support <environments>` with functions like
-  :func:`.find_system_environments` and :func:`.find_virtualenvs`
+.. note:: This documentation is mostly for Plugin developers, who want to
+   improve their editors/IDE with Jedi.
 
 .. _api:
 
-Static Analysis Interface
-~~~~~~~~~~~~~~~~~~~~~~~~~
+The API consists of a few different parts:
 
-.. automodule:: jedi
+- The main starting points for complete/goto: :class:`.Script` and
+  :class:`.Interpreter`. If you work with Jedi you want to understand these
+  classes first.
+- :ref:`API Result Classes <api-classes>`
+- :ref:`Python Versions/Virtualenv Support <environments>` with functions like
+  :func:`.find_system_environments` and :func:`.find_virtualenvs`
+- A way to work with different :ref:`Folders / Projects <projects>`
+- Helpful functions: :func:`.preload_module` and :func:`.set_debug_function`
+
+The methods that you are most likely going to use to work with Jedi are the
+following ones:
+
+.. currentmodule:: jedi
+
+.. autosummary::
+   :nosignatures:
+
+    Script.complete
+    Script.goto
+    Script.infer
+    Script.help
+    Script.get_signatures
+    Script.get_references
+    Script.get_context
+    Script.get_names
+    Script.get_syntax_errors
+    Script.rename
+    Script.inline
+    Script.extract_variable
+    Script.extract_function
+    Script.search
+    Script.complete_search
+    Project.search
+    Project.complete_search
+
+Script
+------
 
 .. autoclass:: jedi.Script
     :members:
+
+Interpreter
+-----------
 .. autoclass:: jedi.Interpreter
     :members:
-.. autofunction:: jedi.preload_module
-.. autofunction:: jedi.set_debug_function
+
+.. _projects:
+
+Projects
+--------
+
+.. automodule:: jedi.api.project
+
+.. autofunction:: jedi.get_default_project
+.. autoclass:: jedi.Project
+    :members:
 
 .. _environments:
 
 Environments
-~~~~~~~~~~~~
+------------
 
 .. automodule:: jedi.api.environment
 
@@ -65,18 +83,31 @@ Environments
 .. autoclass:: jedi.api.environment.Environment
     :members:
 
+Helper Functions
+----------------
+
+.. autofunction:: jedi.preload_module
+.. autofunction:: jedi.set_debug_function
+
+Errors
+------
+
+.. autoexception:: jedi.InternalError
+.. autoexception:: jedi.RefactoringError
+
 Examples
 --------
 
-Completions:
+Completions
+~~~~~~~~~~~
 
 .. sourcecode:: python
 
    >>> import jedi
-   >>> source = '''import json; json.l'''
-   >>> script = jedi.Script(source, path='')
+   >>> code = '''import json; json.l'''
+   >>> script = jedi.Script(code, path='example.py')
    >>> script
-   <jedi.api.Script object at 0x2121b10>
+   <Script: 'example.py' <SameEnvironment: 3.9.0 in /usr>>
    >>> completions = script.complete(1, 19)
    >>> completions
    [<Completion: load>, <Completion: loads>]
@@ -87,12 +118,14 @@ Completions:
    >>> completions[1].name
    'loads'
 
-Definitions / Goto:
+Type Inference / Goto
+~~~~~~~~~~~~~~~~~~~~~
 
 .. sourcecode:: python
 
     >>> import jedi
-    >>> source = '''def my_func():
+    >>> code = '''\
+    ... def my_func():
     ...     print 'called'
     ... 
     ... alias = my_func
@@ -100,30 +133,41 @@ Definitions / Goto:
     ... inception = my_list[2]
     ... 
     ... inception()'''
-    >>> script = jedi.Script(source, path='')
+    >>> script = jedi.Script(code)
     >>>
     >>> script.goto(8, 1)
-    [<Definition inception=my_list[2]>]
+    [<Name full_name='__main__.inception', description='inception = my_list[2]'>]
     >>>
     >>> script.infer(8, 1)
-    [<Definition def my_func>]
+    [<Name full_name='__main__.my_func', description='def my_func'>]
 
-References:
+References
+~~~~~~~~~~
 
 .. sourcecode:: python
 
     >>> import jedi
-    >>> source = '''x = 3
+    >>> code = '''\
+    ... x = 3
     ... if 1 == 2:
     ...     x = 4
     ... else:
     ...     del x'''
-    >>> script = jedi.Script(source, '')
+    >>> script = jedi.Script(code)
     >>> rns = script.get_references(5, 8)
     >>> rns
-    [<Definition full_name='__main__.x', description='x = 3'>,
-     <Definition full_name='__main__.x', description='x'>]
+    [<Name full_name='__main__.x', description='x = 3'>,
+     <Name full_name='__main__.x', description='x = 4'>,
+     <Name full_name='__main__.x', description='del x'>]
     >>> rns[1].line
-    5
-    >>> rns[0].column
-    8
+    3
+    >>> rns[1].column
+    4
+
+Deprecations
+------------
+
+The deprecation process is as follows:
+
+1. A deprecation is announced in any release.
+2. The next major release removes the deprecated functionality.

docs/docs/changelog.rst

@@ -0,0 +1 @@
+.. include:: ../../CHANGELOG.rst

docs/docs/development.rst

@@ -22,16 +22,12 @@ couldn't get rid of complexity. I know that **simple is better than complex**,
 but unfortunately it sometimes requires complex solutions to understand complex
 systems.
 
-Since most of the Jedi internals have been written by me (David Halter), this
-introduction will be written mostly by me, because no one else understands to
-the same level how Jedi works. Actually this is also the reason for exactly this
-part of the documentation. To make multiple people able to edit the Jedi core.
-
-In five chapters I'm trying to describe the internals of |jedi|:
+In six chapters I'm trying to describe the internals of |jedi|:
 
 - :ref:`The Jedi Core <core>`
 - :ref:`Core Extensions <core-extensions>`
 - :ref:`Imports & Modules <imports-modules>`
+- :ref:`Stubs & Annotations <stubs>`
 - :ref:`Caching & Recursions <caching-recursions>`
 - :ref:`Helper modules <dev-helpers>`
 
@@ -59,17 +55,17 @@ because that's where all the magic happens. I need to introduce the :ref:`parser
 Parser
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-Jedi used to have it's internal parser, however this is now a separate project
+Jedi used to have its internal parser, however this is now a separate project
 and is called `parso <http://parso.readthedocs.io>`_.
 
 The parser creates a syntax tree that |jedi| analyses and tries to understand.
-The grammar that this parsers uses is very similar to the official Python
+The grammar that this parser uses is very similar to the official Python
 `grammar files <https://docs.python.org/3/reference/grammar.html>`_.
 
 .. _inference:
 
 Type inference of python code (inference/__init__.py)
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 .. automodule:: jedi.inference
 
@@ -80,7 +76,7 @@ Inference Values (inference/base_value.py)
 
 .. inheritance-diagram::
    jedi.inference.value.instance.TreeInstance
-   jedi.inference.value.klass.Classvalue
+   jedi.inference.value.klass.ClassValue
    jedi.inference.value.function.FunctionValue
    jedi.inference.value.function.FunctionExecutionContext
    :parts: 1
@@ -89,7 +85,7 @@ Inference Values (inference/base_value.py)
 .. _name_resolution:
 
 Name resolution (inference/finder.py)
-++++++++++++++++++++++++++++++++++++
++++++++++++++++++++++++++++++++++++++
 
 .. automodule:: jedi.inference.finder
 
@@ -114,7 +110,7 @@ Core Extensions
 Core Extensions is a summary of the following topics:
 
 - :ref:`Iterables & Dynamic Arrays <iterables>`
-- :ref:`Dynamic Parameters <dynamic>`
+- :ref:`Dynamic Parameters <dynamic_params>`
 - :ref:`Docstrings <docstrings>`
 - :ref:`Refactoring <refactoring>`
 
@@ -125,7 +121,7 @@ without some features.
 .. _iterables:
 
 Iterables & Dynamic Arrays (inference/value/iterable.py)
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 To understand Python on a deeper level, |jedi| needs to understand some of the
 dynamic features of Python like lists that are filled after creation:
@@ -133,33 +129,33 @@ dynamic features of Python like lists that are filled after creation:
 .. automodule:: jedi.inference.value.iterable
 
 
-.. _dynamic:
+.. _dynamic_params:
 
-Parameter completion (inference/dynamic.py)
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Parameter completion (inference/dynamic_params.py)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-.. automodule:: jedi.inference.dynamic
+.. automodule:: jedi.inference.dynamic_params
 
 
 .. _docstrings:
 
 Docstrings (inference/docstrings.py)
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 .. automodule:: jedi.inference.docstrings
 
 .. _refactoring:
 
-Refactoring (inference/refactoring.py)
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Refactoring (api/refactoring.py)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-.. automodule:: jedi.refactoring
+.. automodule:: jedi.api.refactoring
 
 
 .. _imports-modules:
 
 Imports & Modules
--------------------
+-----------------
 
 
 - :ref:`Modules <modules>`
@@ -170,7 +166,7 @@ Imports & Modules
 .. _builtin:
 
 Compiled Modules (inference/compiled.py)
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 .. automodule:: jedi.inference.compiled
 
@@ -178,10 +174,16 @@ Compiled Modules (inference/compiled.py)
 .. _imports:
 
 Imports (inference/imports.py)
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 .. automodule:: jedi.inference.imports
 
+.. _stubs:
+
+Stubs & Annotations (inference/gradual)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. automodule:: jedi.inference.gradual
 
 .. _caching-recursions:
 
@@ -210,13 +212,8 @@ Recursions (recursion.py)
 .. _dev-helpers:
 
 Helper Modules
----------------
+--------------
 
 Most other modules are not really central to how Jedi works. They all contain
 relevant code, but you if you understand the modules above, you pretty much
 understand Jedi.
-
-Python 2/3 compatibility (_compatibility.py)
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. automodule:: jedi._compatibility

docs/docs/features.rst

@@ -1,29 +1,30 @@
 .. include:: ../global.rst
 
-Features and Caveats
-====================
+Features and Limitations
+========================
 
-Jedi obviously supports autocompletion. It's also possible to get it working in
-(:ref:`your REPL (IPython, etc.) <repl-completion>`).
+Jedi's main API calls and features are:
 
-Static analysis is also possible by using ``jedi.Script(...).get_names``.
+- Autocompletion: :meth:`.Script.complete`; It's also possible to get it
+  working in :ref:`your REPL (IPython, etc.) <repl-completion>`
+- Goto/Type Inference: :meth:`.Script.goto` and :meth:`.Script.infer`
+- Static Analysis: :meth:`.Script.get_names` and :meth:`.Script.get_syntax_errors`
+- Refactorings: :meth:`.Script.rename`, :meth:`.Script.inline`,
+  :meth:`.Script.extract_variable` and :meth:`.Script.extract_function`
+- Code Search: :meth:`.Script.search` and :meth:`.Project.search`
 
-Jedi would in theory support refactoring, but we have never publicized it,
-because it's not production ready. If you're interested in helping out here,
-let me know. With the latest parser changes, it should be very easy to actually
-make it work.
+Basic Features
+--------------
 
-
-General Features
-----------------
-
-- Python 2.7 and 3.4+ support
+- Python 3.6+ support
 - Ignores syntax errors and wrong indentation
 - Can deal with complex module / function / class structures
-- Great Virtualenv support
-- Can infer function arguments from sphinx, epydoc and basic numpydoc docstrings,
-  and PEP0484-style type hints (:ref:`type hinting <type-hinting>`)
-- Stub files
+- Great ``virtualenv``/``venv`` support
+- Works great with Python's :ref:`type hinting <type-hinting>`,
+- Understands stub files
+- Can infer function arguments for sphinx, epydoc and basic numpydoc docstrings
+- Is overall a very solid piece of software that has been refined for a long
+  time. Bug reports are very welcome and are usually fixed within a few weeks.
 
 
 Supported Python Features
@@ -38,7 +39,7 @@ Supported Python Features
 - ``*args`` / ``**kwargs``
 - decorators / lambdas / closures
 - generators / iterators
-- some descriptors: property / staticmethod / classmethod
+- descriptors: property / staticmethod / classmethod / custom descriptors
 - some magic methods: ``__call__``, ``__iter__``, ``__next__``, ``__get__``,
   ``__getitem__``, ``__init__``
 - ``list.append()``, ``set.add()``, ``list.extend()``, etc.
@@ -46,191 +47,64 @@ Supported Python Features
 - relative imports
 - ``getattr()`` / ``__getattr__`` / ``__getattribute__``
 - function annotations
-- class decorators (py3k feature, are being ignored too, until I find a use
-  case, that doesn't work with |jedi|)
-- simple/usual ``sys.path`` modifications
+- simple/typical ``sys.path`` modifications
 - ``isinstance`` checks for if/while/assert
 - namespace packages (includes ``pkgutil``, ``pkg_resources`` and PEP420 namespaces)
 - Django / Flask / Buildout support
 - Understands Pytest fixtures
 
 
-Not Supported
--------------
+Limitations
+-----------
 
-Not yet implemented:
+In general Jedi's limit is quite high, but for very big projects or very
+complex code, sometimes Jedi intentionally stops type inference, to avoid
+hanging for a long time.
 
-- manipulations of instances outside the instance variables without using
-  methods
+Additionally there are some Python patterns Jedi does not support. This is
+intentional and below should be a complete list:
 
-Will probably never be implemented:
-
-- metaclasses (how could an auto-completion ever support this)
+- Arbitrary metaclasses: Some metaclasses like enums and dataclasses are
+  reimplemented in Jedi to make them work. Most of the time stubs are good
+  enough to get type inference working, even when metaclasses are involved.
 - ``setattr()``, ``__import__()``
-- writing to some dicts: ``globals()``, ``locals()``, ``object.__dict__``
+- Writing to some dicts: ``globals()``, ``locals()``, ``object.__dict__``
+- Manipulations of instances outside the instance variables without using
+  methods 
 
-
-Caveats
--------
-
-**Slow Performance**
+Performance Issues
+~~~~~~~~~~~~~~~~~~
 
 Importing ``numpy`` can be quite slow sometimes, as well as loading the
-builtins the first time. If you want to speed things up, you could write import
-hooks in |jedi|, which preload stuff. However, once loaded, this is not a
-problem anymore.  The same is true for huge modules like ``PySide``, ``wx``,
-etc.
+builtins the first time. If you want to speed things up, you could preload
+libraries in |jedi|, with :func:`.preload_module`. However, once loaded, this
+should not be a problem anymore.  The same is true for huge modules like
+``PySide``, ``wx``, ``tensorflow``, ``pandas``, etc.
 
-**Security**
+Jedi does not have a very good cache layer. This is probably the biggest and
+only architectural `issue <https://github.com/davidhalter/jedi/issues/1059>`_ in
+Jedi. Unfortunately it is not easy to change that. Dave Halter is thinking
+about rewriting Jedi in Rust, but it has taken Jedi more than 8 years to reach
+version 1.0, a rewrite will probably also take years.
 
-Security is an important issue for |jedi|. Therefore no Python code is
-executed.  As long as you write pure Python, everything is inferred
-statically. But: If you use builtin modules (``c_builtin``) there is no other
-option than to execute those modules. However: Execute isn't that critical (as
-e.g. in pythoncomplete, which used to execute *every* import!), because it
-means one import and no more.  So basically the only dangerous thing is using
-the import itself. If your ``c_builtin`` uses some strange initializations, it
-might be dangerous. But if it does you're screwed anyways, because eventually
-you're going to execute your code, which executes the import.
+Security
+--------
 
+For :class:`.Script`
+~~~~~~~~~~~~~~~~~~~~
 
-Recipes
--------
+Security is an important topic for |jedi|. By default, no code is executed
+within Jedi. As long as you write pure Python, everything is inferred
+statically. If you enable ``load_unsafe_extensions=True`` for your
+:class:`.Project` and you use builtin modules (``c_builtin``) Jedi will execute
+those modules. If you don't trust a code base, please do not enable that
+option. It might lead to arbitrary code execution.
 
-Here are some tips on how to use |jedi| efficiently.
+For :class:`.Interpreter`
+~~~~~~~~~~~~~~~~~~~~~~~~~
 
-
-.. _type-hinting:
-
-Type Hinting
-~~~~~~~~~~~~
-
-If |jedi| cannot detect the type of a function argument correctly (due to the
-dynamic nature of Python), you can help it by hinting the type using
-one of the following docstring/annotation syntax styles:
-
-**PEP-0484 style**
-
-https://www.python.org/dev/peps/pep-0484/
-
-function annotations
-
-::
-
-    def myfunction(node: ProgramNode, foo: str) -> None:
-        """Do something with a ``node``.
-
-        """
-        node.| # complete here
-
-
-assignment, for-loop and with-statement type hints (all Python versions).
-Note that the type hints must be on the same line as the statement
-
-::
-
-    x = foo()  # type: int
-    x, y = 2, 3  # type: typing.Optional[int], typing.Union[int, str] # typing module is mostly supported
-    for key, value in foo.items():  # type: str, Employee  # note that Employee must be in scope
-        pass
-    with foo() as f:  # type: int
-        print(f + 3)
-
-Most of the features in PEP-0484 are supported including the typing module
-(for Python < 3.5 you have to do ``pip install typing`` to use these),
-and forward references.
-
-You can also use stub files.
-
-
-**Sphinx style**
-
-http://www.sphinx-doc.org/en/stable/domains.html#info-field-lists
-
-::
-
-    def myfunction(node, foo):
-        """Do something with a ``node``.
-
-        :type node: ProgramNode
-        :param str foo: foo parameter description
-
-        """
-        node.| # complete here
-
-**Epydoc**
-
-http://epydoc.sourceforge.net/manual-fields.html
-
-::
-
-    def myfunction(node):
-        """Do something with a ``node``.
-
-        @type node: ProgramNode
-
-        """
-        node.| # complete here
-
-**Numpydoc**
-
-https://github.com/numpy/numpy/blob/master/doc/HOWTO_DOCUMENT.rst.txt
-
-In order to support the numpydoc format, you need to install the `numpydoc
-<https://pypi.python.org/pypi/numpydoc>`__ package.
-
-::
-
-    def foo(var1, var2, long_var_name='hi'):
-        r"""A one-line summary that does not use variable names or the
-        function name.
-
-        ...
-
-        Parameters
-        ----------
-        var1 : array_like
-            Array_like means all those objects -- lists, nested lists,
-            etc. -- that can be converted to an array. We can also
-            refer to variables like `var1`.
-        var2 : int
-            The type above can either refer to an actual Python type
-            (e.g. ``int``), or describe the type of the variable in more
-            detail, e.g. ``(N,) ndarray`` or ``array_like``.
-        long_variable_name : {'hi', 'ho'}, optional
-            Choices in brackets, default first when optional.
-
-        ...
-
-        """
-        var2.| # complete here
-
-A little history
-----------------
-
-The Star Wars Jedi are awesome. My Jedi software tries to imitate a little bit
-of the precognition the Jedi have. There's even an awesome `scene
-<https://youtu.be/yHRJLIf7wMU>`_ of Monty Python Jedis :-).
-
-But actually the name hasn't so much to do with Star Wars. It's part of my
-second name.
-
-After I explained Guido van Rossum, how some parts of my auto-completion work,
-he said (we drank a beer or two):
-
-    *"Oh, that worries me..."*
-
-When it's finished, I hope he'll like it :-)
-
-I actually started Jedi, because there were no good solutions available for VIM.
-Most auto-completions just didn't work well. The only good solution was PyCharm.
-But I like my good old VIM. Rope was never really intended to be an
-auto-completion (and also I really hate project folders for my Python scripts).
-It's more of a refactoring suite. So I decided to do my own version of a
-completion, which would execute non-dangerous code. But I soon realized, that
-this wouldn't work. So I built an extremely recursive thing which understands
-many of Python's key features.
-
-By the way, I really tried to program it as understandable as possible. But I
-think understanding it might need quite some time, because of its recursive
-nature.
+If you want security for :class:`.Interpreter`, ``do not`` use it. Jedi does
+execute properties and in general is not very careful to avoid code execution.
+This is intentional: Most people trust the code bases they have imported,
+because at that point a malicious code base would have had code execution
+already.

docs/docs/installation.rst

@@ -3,6 +3,15 @@
 Installation and Configuration
 ==============================
 
+.. warning:: Most people will want to install Jedi as a submodule/vendored and
+   not through pip/system wide. The reason for this is that it makes sense that
+   the plugin that uses Jedi has always access to it. Otherwise Jedi will not
+   work properly when virtualenvs are activated. So please read the
+   documentation of your editor/IDE plugin to install Jedi.
+
+   For plugin developers, Jedi works best if it is always available. Vendoring
+   is a pretty good option for that.
+
 You can either include |jedi| as a submodule in your text editor plugin (like
 jedi-vim_ does by default), or you can install it systemwide.
 
@@ -41,14 +50,6 @@ Arch Linux
 You can install |jedi| directly from official Arch Linux packages:
 
 - `python-jedi <https://www.archlinux.org/packages/community/any/python-jedi/>`__
-  (Python 3)
-- `python2-jedi <https://www.archlinux.org/packages/community/any/python2-jedi/>`__
-  (Python 2)
-
-The specified Python version just refers to the *runtime environment* for
-|jedi|. Use the Python 2 version if you're running vim (or whatever editor you
-use) under Python 2. Otherwise, use the Python 3 version. But whatever version
-you choose, both are able to complete both Python 2 and 3 *code*.
 
 (There is also a packaged version of the vim plugin available: 
 `vim-jedi at Arch Linux <https://www.archlinux.org/packages/community/any/vim-jedi/>`__.)

docs/docs/testing.rst

@@ -3,21 +3,17 @@
 Jedi Testing
 ============
 
-The test suite depends on ``tox`` and ``pytest``::
+The test suite depends on ``pytest``::
 
-    pip install tox pytest
+    pip install pytest
 
-To run the tests for all supported Python versions::
-
-    tox
-
-If you want to test only a specific Python version (e.g. Python 2.7), it's as
+If you want to test only a specific Python version (e.g. Python 3.8), it is as
 easy as::
 
-    tox -e py27
+    python3.8 -m pytest
 
-Tests are also run automatically on `Travis CI
-<https://travis-ci.org/davidhalter/jedi/>`_.
+Tests are also run automatically on `GitHub Actions
+<https://github.com/davidhalter/jedi/actions>`_.
 
 You want to add a test for |jedi|? Great! We love that. Normally you should
 write your tests as :ref:`Blackbox Tests <blackbox>`. Most tests would
@@ -28,8 +24,8 @@ simple and readable testing structure.
 
 .. _blackbox:
 
-Blackbox Tests (run.py)
-~~~~~~~~~~~~~~~~~~~~~~~
+Integration Tests (run.py)
+~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 .. automodule:: test.run
 

docs/docs/usage.rst

@@ -1,79 +1,109 @@
 .. include:: ../global.rst
 
-End User Usage
-==============
+Using Jedi
+==========
 
-If you are a not an IDE Developer, the odds are that you just want to use
-|jedi| as a browser plugin or in the shell. Yes that's :ref:`also possible
-<repl-completion>`!
+|jedi| is can be used with a variety of :ref:`plugins <editor-plugins>`,
+:ref:`language servers <language-servers>` and other software.
+It is also possible to use |jedi| in the :ref:`Python shell or with IPython
+<repl-completion>`.
 
-|jedi| is relatively young and can be used in a variety of Plugins and
-Software. If your Editor/IDE is not among them, recommend |jedi| to your IDE
-developers.
+Below you can also find a list of :ref:`recipes for type hinting <recipes>`.
 
+.. _language-servers:
+
+Language Servers
+--------------
+
+- `jedi-language-server <https://github.com/pappasam/jedi-language-server>`_
+- `python-language-server <https://github.com/palantir/python-language-server>`_ (currently unmaintained)
+- `python-lsp-server <https://github.com/python-lsp/python-lsp-server>`_ (fork from python-language-server)
+- `anakin-language-server <https://github.com/muffinmad/anakin-language-server>`_
 
 .. _editor-plugins:
 
 Editor Plugins
 --------------
 
-Vim:
+Vim
+~~~
 
 - jedi-vim_
 - YouCompleteMe_
 - deoplete-jedi_
 
-Emacs:
+Visual Studio Code
+~~~~~~~~~~~~~~~~~~
+
+- `Python Extension`_
+
+Emacs
+~~~~~
 
 - Jedi.el_
 - elpy_
 - anaconda-mode_
 
-Sublime Text 2/3:
+Sublime Text 2/3
+~~~~~~~~~~~~~~~~
 
 - SublimeJEDI_ (ST2 & ST3)
 - anaconda_ (only ST3)
 
-SynWrite:
+SynWrite
+~~~~~~~~
 
 - SynJedi_
 
-TextMate:
+TextMate
+~~~~~~~~
 
 - Textmate_ (Not sure if it's actually working)
 
-Kate:
+Kate
+~~~~
 
 - Kate_ version 4.13+ `supports it natively
   <https://projects.kde.org/projects/kde/applications/kate/repository/entry/addons/kate/pate/src/plugins/python_autocomplete_jedi.py?rev=KDE%2F4.13>`__,
   you have to enable it, though.
 
-Visual Studio Code:
-
-- `Python Extension`_
-
-Atom:
+Atom
+~~~~
 
 - autocomplete-python-jedi_
 
-GNOME Builder:
+GNOME Builder
+~~~~~~~~~~~~~
 
 - `GNOME Builder`_ `supports it natively
   <https://git.gnome.org/browse/gnome-builder/tree/plugins/jedi>`__,
   and is enabled by default.
 
-Gedit:
+Gedit
+~~~~~
 
 - gedi_
 
-Eric IDE:
+Eric IDE
+~~~~~~~~
 
-- `Eric IDE`_ (Available as a plugin)
+- `Eric IDE`_
 
-Web Debugger:
+Web Debugger
+~~~~~~~~~~~~
 
 - wdb_
 
+xonsh shell
+~~~~~~~~~~~
+
+Jedi is a preinstalled extension in `xonsh shell <https://xon.sh/contents.html>`_. 
+Run the following command to enable:
+
+::
+
+    xontrib load jedi
+
 and many more!
 
 .. _repl-completion:
@@ -81,11 +111,14 @@ and many more!
 Tab Completion in the Python Shell
 ----------------------------------
 
-Starting with Ipython `6.0.0` Jedi is a dependency of IPython. Autocompletion
-in IPython is therefore possible without additional configuration.
+Jedi is a dependency of IPython. Autocompletion in IPython is therefore
+possible without additional configuration.
+
+Here is an `example video <https://vimeo.com/122332037>`_ how REPL completion
+can look like in a different shell.
 
 There are two different options how you can use Jedi autocompletion in
-your Python interpreter. One with your custom ``$HOME/.pythonrc.py`` file
+your ``python`` interpreter. One with your custom ``$HOME/.pythonrc.py`` file
 and one that uses ``PYTHONSTARTUP``.
 
 Using ``PYTHONSTARTUP``
@@ -93,11 +126,137 @@ Using ``PYTHONSTARTUP``
 
 .. automodule:: jedi.api.replstartup
 
-Using a custom ``$HOME/.pythonrc.py``
+Using a Custom ``$HOME/.pythonrc.py``
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 .. autofunction:: jedi.utils.setup_readline
 
+.. _recipes:
+
+Recipes
+-------
+
+Here are some tips on how to use |jedi| efficiently.
+
+
+.. _type-hinting:
+
+Type Hinting
+~~~~~~~~~~~~
+
+If |jedi| cannot detect the type of a function argument correctly (due to the
+dynamic nature of Python), you can help it by hinting the type using
+one of the docstring/annotation styles below. **Only gradual typing will
+always work**, all the docstring solutions are glorified hacks and more
+complicated cases will probably not work.
+
+Official Gradual Typing (Recommended)
++++++++++++++++++++++++++++++++++++++
+
+You can read a lot about Python's gradual typing system in the corresponding
+PEPs like:
+
+- `PEP 484 <https://www.python.org/dev/peps/pep-0484/>`_ as an introduction
+- `PEP 526 <https://www.python.org/dev/peps/pep-0526/>`_ for variable annotations
+- `PEP 589 <https://www.python.org/dev/peps/pep-0589/>`_ for ``TypeDict``
+- There are probably more :)
+
+Below you can find a few examples how you can use this feature.
+
+Function annotations::
+
+    def myfunction(node: ProgramNode, foo: str) -> None:
+        """Do something with a ``node``.
+
+        """
+        node.| # complete here
+
+
+Assignment, for-loop and with-statement type hints::
+
+    import typing
+    x: int = foo()
+    y: typing.Optional[int] = 3
+
+    key: str
+    value: Employee
+    for key, value in foo.items():
+        pass
+
+    f: Union[int, float]
+    with foo() as f:
+        print(f + 3)
+
+PEP-0484 should be supported in its entirety. Feel free to open issues if that
+is not the case. You can also use stub files.
+
+
+Sphinx style
+++++++++++++
+
+http://www.sphinx-doc.org/en/stable/domains.html#info-field-lists
+
+::
+
+    def myfunction(node, foo):
+        """
+        Do something with a ``node``.
+
+        :type node: ProgramNode
+        :param str foo: foo parameter description
+        """
+        node.| # complete here
+
+Epydoc
+++++++
+
+http://epydoc.sourceforge.net/manual-fields.html
+
+::
+
+    def myfunction(node):
+        """
+        Do something with a ``node``.
+
+        @type node: ProgramNode
+        """
+        node.| # complete here
+
+Numpydoc
+++++++++
+
+https://github.com/numpy/numpy/blob/master/doc/HOWTO_DOCUMENT.rst.txt
+
+In order to support the numpydoc format, you need to install the `numpydoc
+<https://pypi.python.org/pypi/numpydoc>`__ package.
+
+::
+
+    def foo(var1, var2, long_var_name='hi'):
+        r"""
+        A one-line summary that does not use variable names or the
+        function name.
+
+        ...
+
+        Parameters
+        ----------
+        var1 : array_like
+            Array_like means all those objects -- lists, nested lists,
+            etc. -- that can be converted to an array. We can also
+            refer to variables like `var1`.
+        var2 : int
+            The type above can either refer to an actual Python type
+            (e.g. ``int``), or describe the type of the variable in more
+            detail, e.g. ``(N,) ndarray`` or ``array_like``.
+        long_variable_name : {'hi', 'ho'}, optional
+            Choices in brackets, default first when optional.
+
+        ...
+
+        """
+        var2.| # complete here
+
 .. _jedi-vim: https://github.com/davidhalter/jedi-vim
 .. _youcompleteme: https://valloric.github.io/YouCompleteMe/
 .. _deoplete-jedi: https://github.com/zchee/deoplete-jedi
@@ -114,4 +273,4 @@ Using a custom ``$HOME/.pythonrc.py``
 .. _GNOME Builder: https://wiki.gnome.org/Apps/Builder/
 .. _gedi: https://github.com/isamert/gedi
 .. _Eric IDE: https://eric-ide.python-projects.org
-.. _Python Extension: https://marketplace.visualstudio.com/items?itemName=donjayamanne.python
+.. _Python Extension: https://marketplace.visualstudio.com/items?itemName=ms-python.python

docs/global.rst

@@ -1,3 +1,3 @@
 :orphan:
 
-.. |jedi| replace:: *Jedi*
+.. |jedi| replace:: Jedi

docs/index.rst

@@ -1,13 +1,40 @@
 .. include global.rst
 
-Jedi - an awesome autocompletion/static analysis library for Python
-===================================================================
+.. meta::
+    :github_url: https://github.com/davidhalter/jedi
 
-Release v\ |release|. (:doc:`Installation <docs/installation>`)
+Jedi - an awesome autocompletion, static analysis and refactoring library for Python
+====================================================================================
+
+.. image:: https://img.shields.io/github/stars/davidhalter/jedi.svg?style=social&label=Star&maxAge=2592000
+    :target: https://github.com/davidhalter/jedi
+    :alt: GitHub stars
+
+.. image:: http://isitmaintained.com/badge/open/davidhalter/jedi.svg
+    :target: https://github.com/davidhalter/jedi/issues
+    :alt: The percentage of open issues and pull requests
+
+.. image:: http://isitmaintained.com/badge/resolution/davidhalter/jedi.svg
+    :target: https://github.com/davidhalter/jedi/issues
+    :alt: The resolution time is the median time an issue or pull request stays open.
+
+.. image:: https://github.com/davidhalter/jedi/workflows/ci/badge.svg?branch=master
+    :target: https://github.com/davidhalter/jedi/actions
+    :alt: Tests
+
+.. image:: https://coveralls.io/repos/davidhalter/jedi/badge.svg?branch=master
+    :target: https://coveralls.io/r/davidhalter/jedi
+    :alt: Coverage status
+
+.. image:: https://pepy.tech/badge/jedi
+    :target: https://pepy.tech/project/jedi
+    :alt: PyPI Downloads
+
+`Github Repository <https://github.com/davidhalter/jedi>`_
 
 .. automodule:: jedi
 
-Autocompletion can look like this (e.g. VIM plugin):
+Autocompletion can for example look like this in jedi-vim:
 
 .. figure:: _screenshots/screenshot_complete.png
 
@@ -18,16 +45,18 @@ Docs
 ----
 
 .. toctree::
-   :maxdepth: 2
+   :maxdepth: 1
 
    docs/usage
-   docs/installation
    docs/features
    docs/api
    docs/api-classes
+   docs/installation
    docs/settings
    docs/development
    docs/testing
+   docs/acknowledgements
+   docs/changelog
 
 
 .. _resources:
@@ -35,6 +64,9 @@ Docs
 Resources
 ---------
 
+If you want to stay **up-to-date** with releases, please **subscribe** to this
+mailing list: https://groups.google.com/g/jedi-announce. To subscribe you can
+simply send an empty email to ``jedi-announce+subscribe@googlegroups.com``.
+
 - `Source Code on Github <https://github.com/davidhalter/jedi>`_
-- `Travis Testing <https://travis-ci.org/davidhalter/jedi>`_
 - `Python Package Index <https://pypi.python.org/pypi/jedi/>`_

jedi/__init__.py

@@ -1,16 +1,13 @@
 """
-Jedi is a static analysis tool for Python that can be used in IDEs/editors.
-Jedi has a focus on autocompletion and goto functionality. Jedi is fast and is
-very well tested. It understands Python and stubs on a deep level.
+Jedi is a static analysis tool for Python that is typically used in
+IDEs/editors plugins. Jedi has a focus on autocompletion and goto
+functionality. Other features include refactoring, code search and finding
+references.
 
-Jedi has support for different goto functions. It's possible to search for
-references and list names in a Python file to get information about them.
-
-Jedi uses a very simple API to connect with IDE's. There's a reference
-implementation as a `VIM-Plugin <https://github.com/davidhalter/jedi-vim>`_,
-which uses Jedi's autocompletion.  We encourage you to use Jedi in your IDEs.
-Autocompletion in your REPL is also possible, IPython uses it natively and for
-the CPython REPL you have to install it.
+Jedi has a simple API to work with. There is a reference implementation as a
+`VIM-Plugin <https://github.com/davidhalter/jedi-vim>`_. Autocompletion in your
+REPL is also possible, IPython uses it natively and for the CPython REPL you
+can install it. Jedi is well tested and bugs should be rare.
 
 Here's a simple example of the autocompletion feature:
 
@@ -28,20 +25,18 @@ Here's a simple example of the autocompletion feature:
 ad
 >>> print(completions[0].name)
 load
-
-As you see Jedi is pretty simple and allows you to concentrate on writing a
-good text editor, while still having very good IDE features for Python.
 """
 
-__version__ = '0.16.0'
+__version__ = '0.19.1'
 
-from jedi.api import Script, Interpreter, set_debug_function, \
-    preload_module, names
+from jedi.api import Script, Interpreter, set_debug_function, preload_module
 from jedi import settings
 from jedi.api.environment import find_virtualenvs, find_system_environments, \
     get_default_environment, InvalidPythonEnvironment, create_environment, \
-    get_system_environment
-from jedi.api.exceptions import InternalError
+    get_system_environment, InterpreterEnvironment
+from jedi.api.project import Project, get_default_project
+from jedi.api.exceptions import InternalError, RefactoringError
+
 # Finally load the internal plugins. This is only internal.
 from jedi.plugins import registry
 del registry

jedi/__main__.py

@@ -27,8 +27,8 @@ def _start_linter():
             paths = [path]
 
         try:
-            for path in paths:
-                for error in jedi.Script(path=path)._analysis():
+            for p in paths:
+                for error in jedi.Script(path=p)._analysis():
                     print(error)
         except Exception:
             if '--pdb' in sys.argv:
@@ -44,19 +44,29 @@ def _complete():
     import jedi
     import pdb
 
+    if '-d' in sys.argv:
+        sys.argv.remove('-d')
+        jedi.set_debug_function()
+
     try:
-        for c in jedi.Script(sys.argv[2]).complete():
+        completions = jedi.Script(sys.argv[2]).complete()
+        for c in completions:
             c.docstring()
             c.type
-    except Exception:
+    except Exception as e:
+        print(repr(e))
         pdb.post_mortem()
+    else:
+        print(completions)
 
 
 if len(sys.argv) == 2 and sys.argv[1] == 'repl':
     # don't want to use __main__ only for repl yet, maybe we want to use it for
     # something else. So just use the keyword ``repl`` for now.
     print(join(dirname(abspath(__file__)), 'api', 'replstartup.py'))
-elif len(sys.argv) > 1 and sys.argv[1] == 'linter':
+elif len(sys.argv) > 1 and sys.argv[1] == '_linter':
     _start_linter()
 elif len(sys.argv) > 1 and sys.argv[1] == '_complete':
     _complete()
+else:
+    print('Command not implemented: %s' % sys.argv[1])

jedi/_compatibility.py

@@ -1,503 +1,28 @@
 """
-To ensure compatibility from Python ``2.7`` - ``3.x``, a module has been
-created. Clearly there is huge need to use conforming syntax.
+This module is here to ensure compatibility of Windows/Linux/MacOS and
+different Python versions.
 """
-from __future__ import print_function
-import atexit
 import errno
-import functools
 import sys
-import os
-import re
-import pkgutil
-import warnings
-import inspect
-import subprocess
-import weakref
-try:
-    import importlib
-except ImportError:
-    pass
-from zipimport import zipimporter
-
-from jedi.file_io import KnownContentFileIO, ZipFileIO
-
-is_py3 = sys.version_info[0] >= 3
-is_py35 = is_py3 and sys.version_info[1] >= 5
-py_version = int(str(sys.version_info[0]) + str(sys.version_info[1]))
-
-
-if sys.version_info[:2] < (3, 5):
-    """
-    A super-minimal shim around listdir that behave like
-    scandir for the information we need.
-    """
-    class _DirEntry:
-
-        def __init__(self, name, basepath):
-            self.name = name
-            self.basepath = basepath
-
-        def is_dir(self):
-            path_for_name = os.path.join(self.basepath, self.name)
-            return os.path.isdir(path_for_name)
-
-    def scandir(dir):
-        return [_DirEntry(name, dir) for name in os.listdir(dir)]
-else:
-    from os import scandir
-
-
-class DummyFile(object):
-    def __init__(self, loader, string):
-        self.loader = loader
-        self.string = string
-
-    def read(self):
-        return self.loader.get_source(self.string)
-
-    def close(self):
-        del self.loader
-
-
-def find_module_py34(string, path=None, full_name=None, is_global_search=True):
-    spec = None
-    loader = None
-
-    for finder in sys.meta_path:
-        if is_global_search and finder != importlib.machinery.PathFinder:
-            p = None
-        else:
-            p = path
-        try:
-            find_spec = finder.find_spec
-        except AttributeError:
-            # These are old-school clases that still have a different API, just
-            # ignore those.
-            continue
-
-        spec = find_spec(string, p)
-        if spec is not None:
-            loader = spec.loader
-            if loader is None and not spec.has_location:
-                # This is a namespace package.
-                full_name = string if not path else full_name
-                implicit_ns_info = ImplicitNSInfo(full_name, spec.submodule_search_locations._path)
-                return implicit_ns_info, True
-            break
-
-    return find_module_py33(string, path, loader)
-
-
-def find_module_py33(string, path=None, loader=None, full_name=None, is_global_search=True):
-    loader = loader or importlib.machinery.PathFinder.find_module(string, path)
-
-    if loader is None and path is None:  # Fallback to find builtins
-        try:
-            with warnings.catch_warnings(record=True):
-                # Mute "DeprecationWarning: Use importlib.util.find_spec()
-                # instead." While we should replace that in the future, it's
-                # probably good to wait until we deprecate Python 3.3, since
-                # it was added in Python 3.4 and find_loader hasn't been
-                # removed in 3.6.
-                loader = importlib.find_loader(string)
-        except ValueError as e:
-            # See #491. Importlib might raise a ValueError, to avoid this, we
-            # just raise an ImportError to fix the issue.
-            raise ImportError("Originally  " + repr(e))
-
-    if loader is None:
-        raise ImportError("Couldn't find a loader for {}".format(string))
-
-    return _from_loader(loader, string)
-
-
-def _from_loader(loader, string):
-    is_package = loader.is_package(string)
-    try:
-        get_filename = loader.get_filename
-    except AttributeError:
-        return None, is_package
-    else:
-        module_path = cast_path(get_filename(string))
-
-    # To avoid unicode and read bytes, "overwrite" loader.get_source if
-    # possible.
-    f = type(loader).get_source
-    if is_py3 and f is not importlib.machinery.SourceFileLoader.get_source:
-        # Unfortunately we are reading unicode here, not bytes.
-        # It seems hard to get bytes, because the zip importer
-        # logic just unpacks the zip file and returns a file descriptor
-        # that we cannot as easily access. Therefore we just read it as
-        # a string in the cases where get_source was overwritten.
-        code = loader.get_source(string)
-    else:
-        code = _get_source(loader, string)
-
-    if code is None:
-        return None, is_package
-    if isinstance(loader, zipimporter):
-        return ZipFileIO(module_path, code, cast_path(loader.archive)), is_package
-
-    return KnownContentFileIO(module_path, code), is_package
-
-
-def _get_source(loader, fullname):
-    """
-    This method is here as a replacement for SourceLoader.get_source. That
-    method returns unicode, but we prefer bytes.
-    """
-    path = loader.get_filename(fullname)
-    try:
-        return loader.get_data(path)
-    except OSError:
-        raise ImportError('source not available through get_data()',
-                          name=fullname)
-
-
-def find_module_pre_py3(string, path=None, full_name=None, is_global_search=True):
-    # This import is here, because in other places it will raise a
-    # DeprecationWarning.
-    import imp
-    try:
-        module_file, module_path, description = imp.find_module(string, path)
-        module_type = description[2]
-        is_package = module_type is imp.PKG_DIRECTORY
-        if is_package:
-            # In Python 2 directory package imports are returned as folder
-            # paths, not __init__.py paths.
-            p = os.path.join(module_path, '__init__.py')
-            try:
-                module_file = open(p)
-                module_path = p
-            except FileNotFoundError:
-                pass
-        elif module_type != imp.PY_SOURCE:
-            if module_file is not None:
-                module_file.close()
-            module_file = None
-
-        if module_file is None:
-            code = None
-            return None, is_package
-
-        with module_file:
-            code = module_file.read()
-        return KnownContentFileIO(cast_path(module_path), code), is_package
-    except ImportError:
-        pass
-
-    if path is None:
-        path = sys.path
-    for item in path:
-        loader = pkgutil.get_importer(item)
-        if loader:
-            loader = loader.find_module(string)
-            if loader is not None:
-                return _from_loader(loader, string)
-    raise ImportError("No module named {}".format(string))
-
-
-find_module = find_module_py34 if is_py3 else find_module_pre_py3
-find_module.__doc__ = """
-Provides information about a module.
-
-This function isolates the differences in importing libraries introduced with
-python 3.3 on; it gets a module name and optionally a path. It will return a
-tuple containin an open file for the module (if not builtin), the filename
-or the name of the module if it is a builtin one and a boolean indicating
-if the module is contained in a package.
-"""
-
-
-def _iter_modules(paths, prefix=''):
-    # Copy of pkgutil.iter_modules adapted to work with namespaces
-
-    for path in paths:
-        importer = pkgutil.get_importer(path)
-
-        if not isinstance(importer, importlib.machinery.FileFinder):
-            # We're only modifying the case for FileFinder. All the other cases
-            # still need to be checked (like zip-importing). Do this by just
-            # calling the pkgutil version.
-            for mod_info in pkgutil.iter_modules([path], prefix):
-                yield mod_info
-            continue
-
-        # START COPY OF pkutils._iter_file_finder_modules.
-        if importer.path is None or not os.path.isdir(importer.path):
-            return
-
-        yielded = {}
-
-        try:
-            filenames = os.listdir(importer.path)
-        except OSError:
-            # ignore unreadable directories like import does
-            filenames = []
-        filenames.sort()  # handle packages before same-named modules
-
-        for fn in filenames:
-            modname = inspect.getmodulename(fn)
-            if modname == '__init__' or modname in yielded:
-                continue
-
-            # jedi addition: Avoid traversing special directories
-            if fn.startswith('.') or fn == '__pycache__':
-                continue
-
-            path = os.path.join(importer.path, fn)
-            ispkg = False
-
-            if not modname and os.path.isdir(path) and '.' not in fn:
-                modname = fn
-                # A few jedi modifications: Don't check if there's an
-                # __init__.py
-                try:
-                    os.listdir(path)
-                except OSError:
-                    # ignore unreadable directories like import does
-                    continue
-                ispkg = True
-
-            if modname and '.' not in modname:
-                yielded[modname] = 1
-                yield importer, prefix + modname, ispkg
-        # END COPY
-
-
-iter_modules = _iter_modules if py_version >= 34 else pkgutil.iter_modules
-
-
-class ImplicitNSInfo(object):
-    """Stores information returned from an implicit namespace spec"""
-    def __init__(self, name, paths):
-        self.name = name
-        self.paths = paths
-
-
-if is_py3:
-    all_suffixes = importlib.machinery.all_suffixes
-else:
-    def all_suffixes():
-        # Is deprecated and raises a warning in Python 3.6.
-        import imp
-        return [suffix for suffix, _, _ in imp.get_suffixes()]
-
-
-# unicode function
-try:
-    unicode = unicode
-except NameError:
-    unicode = str
-
-
-# re-raise function
-if is_py3:
-    def reraise(exception, traceback):
-        raise exception.with_traceback(traceback)
-else:
-    eval(compile("""
-def reraise(exception, traceback):
-    raise exception, None, traceback
-""", 'blub', 'exec'))
-
-reraise.__doc__ = """
-Re-raise `exception` with a `traceback` object.
-
-Usage::
-
-    reraise(Exception, sys.exc_info()[2])
-
-"""
-
-
-def use_metaclass(meta, *bases):
-    """ Create a class with a metaclass. """
-    if not bases:
-        bases = (object,)
-    return meta("Py2CompatibilityMetaClass", bases, {})
-
-
-try:
-    encoding = sys.stdout.encoding
-    if encoding is None:
-        encoding = 'utf-8'
-except AttributeError:
-    encoding = 'ascii'
-
-
-def u(string, errors='strict'):
-    """Cast to unicode DAMMIT!
-    Written because Python2 repr always implicitly casts to a string, so we
-    have to cast back to a unicode (and we now that we always deal with valid
-    unicode, because we check that in the beginning).
-    """
-    if isinstance(string, bytes):
-        return unicode(string, encoding='UTF-8', errors=errors)
-    return string
-
-
-def cast_path(obj):
-    """
-    Take a bytes or str path and cast it to unicode.
-
-    Apparently it is perfectly fine to pass both byte and unicode objects into
-    the sys.path. This probably means that byte paths are normal at other
-    places as well.
-
-    Since this just really complicates everything and Python 2.7 will be EOL
-    soon anyway, just go with always strings.
-    """
-    return u(obj, errors='replace')
-
-
-def force_unicode(obj):
-    # Intentionally don't mix those two up, because those two code paths might
-    # be different in the future (maybe windows?).
-    return cast_path(obj)
-
-
-try:
-    import builtins  # module name in python 3
-except ImportError:
-    import __builtin__ as builtins  # noqa: F401
-
-
-import ast  # noqa: F401
-
-
-def literal_eval(string):
-    return ast.literal_eval(string)
-
-
-try:
-    from itertools import zip_longest
-except ImportError:
-    from itertools import izip_longest as zip_longest  # Python 2  # noqa: F401
-
-try:
-    FileNotFoundError = FileNotFoundError
-except NameError:
-    FileNotFoundError = IOError
-
-try:
-    IsADirectoryError = IsADirectoryError
-except NameError:
-    IsADirectoryError = IOError
-
-try:
-    PermissionError = PermissionError
-except NameError:
-    PermissionError = IOError
-
-
-def no_unicode_pprint(dct):
-    """
-    Python 2/3 dict __repr__ may be different, because of unicode differens
-    (with or without a `u` prefix). Normally in doctests we could use `pprint`
-    to sort dicts and check for equality, but here we have to write a separate
-    function to do that.
-    """
-    import pprint
-    s = pprint.pformat(dct)
-    print(re.sub("u'", "'", s))
-
-
-def utf8_repr(func):
-    """
-    ``__repr__`` methods in Python 2 don't allow unicode objects to be
-    returned. Therefore cast them to utf-8 bytes in this decorator.
-    """
-    def wrapper(self):
-        result = func(self)
-        if isinstance(result, unicode):
-            return result.encode('utf-8')
-        else:
-            return result
-
-    if is_py3:
-        return func
-    else:
-        return wrapper
-
-
-if is_py3:
-    import queue
-else:
-    import Queue as queue  # noqa: F401
-
-try:
-    # Attempt to load the C implementation of pickle on Python 2 as it is way
-    # faster.
-    import cPickle as pickle
-except ImportError:
-    import pickle
-if sys.version_info[:2] == (3, 3):
-    """
-    Monkeypatch the unpickler in Python 3.3. This is needed, because the
-    argument `encoding='bytes'` is not supported in 3.3, but badly needed to
-    communicate with Python 2.
-    """
-
-    class NewUnpickler(pickle._Unpickler):
-        dispatch = dict(pickle._Unpickler.dispatch)
-
-        def _decode_string(self, value):
-            # Used to allow strings from Python 2 to be decoded either as
-            # bytes or Unicode strings.  This should be used only with the
-            # STRING, BINSTRING and SHORT_BINSTRING opcodes.
-            if self.encoding == "bytes":
-                return value
-            else:
-                return value.decode(self.encoding, self.errors)
-
-        def load_string(self):
-            data = self.readline()[:-1]
-            # Strip outermost quotes
-            if len(data) >= 2 and data[0] == data[-1] and data[0] in b'"\'':
-                data = data[1:-1]
-            else:
-                raise pickle.UnpicklingError("the STRING opcode argument must be quoted")
-            self.append(self._decode_string(pickle.codecs.escape_decode(data)[0]))
-        dispatch[pickle.STRING[0]] = load_string
-
-        def load_binstring(self):
-            # Deprecated BINSTRING uses signed 32-bit length
-            len, = pickle.struct.unpack('<i', self.read(4))
-            if len < 0:
-                raise pickle.UnpicklingError("BINSTRING pickle has negative byte count")
-            data = self.read(len)
-            self.append(self._decode_string(data))
-        dispatch[pickle.BINSTRING[0]] = load_binstring
-
-        def load_short_binstring(self):
-            len = self.read(1)[0]
-            data = self.read(len)
-            self.append(self._decode_string(data))
-        dispatch[pickle.SHORT_BINSTRING[0]] = load_short_binstring
-
-    def load(file, fix_imports=True, encoding="ASCII", errors="strict"):
-        return NewUnpickler(file, fix_imports=fix_imports,
-                            encoding=encoding, errors=errors).load()
-
-    def loads(s, fix_imports=True, encoding="ASCII", errors="strict"):
-        if isinstance(s, str):
-            raise TypeError("Can't load pickle from unicode string")
-        file = pickle.io.BytesIO(s)
-        return NewUnpickler(file, fix_imports=fix_imports,
-                            encoding=encoding, errors=errors).load()
-
-    pickle.Unpickler = NewUnpickler
-    pickle.load = load
-    pickle.loads = loads
+import pickle
+from typing import Any
+
+
+class Unpickler(pickle.Unpickler):
+    def find_class(self, module: str, name: str) -> Any:
+        # Python 3.13 moved pathlib implementation out of __init__.py as part of
+        # generalising its implementation. Ensure that we support loading
+        # pickles from 3.13 on older version of Python. Since 3.13 maintained a
+        # compatible API, pickles from older Python work natively on the newer
+        # version.
+        if module == 'pathlib._local':
+            module = 'pathlib'
+        return super().find_class(module, name)
 
 
 def pickle_load(file):
     try:
-        if is_py3:
-            return pickle.load(file, encoding='bytes')
-        return pickle.load(file)
+        return Unpickler(file).load()
     # Python on Windows don't throw EOF errors for pipes. So reraise them with
     # the correct type, which is caught upwards.
     except OSError:
@@ -506,24 +31,8 @@ def pickle_load(file):
         raise
 
 
-def _python2_dct_keys_to_unicode(data):
-    """
-    Python 2 stores object __dict__ entries as bytes, not unicode, correct it
-    here. Python 2 can deal with both, Python 3 expects unicode.
-    """
-    if isinstance(data, tuple):
-        return tuple(_python2_dct_keys_to_unicode(x) for x in data)
-    elif isinstance(data, list):
-        return list(_python2_dct_keys_to_unicode(x) for x in data)
-    elif hasattr(data, '__dict__') and type(data.__dict__) == dict:
-        data.__dict__ = {unicode(k): v for k, v in data.__dict__.items()}
-    return data
-
-
 def pickle_dump(data, file, protocol):
     try:
-        if not is_py3:
-            data = _python2_dct_keys_to_unicode(data)
         pickle.dump(data, file, protocol)
         # On Python 3.3 flush throws sometimes an error even though the writing
         # operation should be completed.
@@ -534,201 +43,3 @@ def pickle_dump(data, file, protocol):
         if sys.platform == 'win32':
             raise IOError(errno.EPIPE, "Broken pipe")
         raise
-
-
-# Determine the highest protocol version compatible for a given list of Python
-# versions.
-def highest_pickle_protocol(python_versions):
-    protocol = 4
-    for version in python_versions:
-        if version[0] == 2:
-            # The minimum protocol version for the versions of Python that we
-            # support (2.7 and 3.3+) is 2.
-            return 2
-        if version[1] < 4:
-            protocol = 3
-    return protocol
-
-
-try:
-    from inspect import Parameter
-except ImportError:
-    class Parameter(object):
-        POSITIONAL_ONLY = object()
-        POSITIONAL_OR_KEYWORD = object()
-        VAR_POSITIONAL = object()
-        KEYWORD_ONLY = object()
-        VAR_KEYWORD = object()
-
-
-class GeneralizedPopen(subprocess.Popen):
-    def __init__(self, *args, **kwargs):
-        if os.name == 'nt':
-            try:
-                # Was introduced in Python 3.7.
-                CREATE_NO_WINDOW = subprocess.CREATE_NO_WINDOW
-            except AttributeError:
-                CREATE_NO_WINDOW = 0x08000000
-            kwargs['creationflags'] = CREATE_NO_WINDOW
-        # The child process doesn't need file descriptors except 0, 1, 2.
-        # This is unix only.
-        kwargs['close_fds'] = 'posix' in sys.builtin_module_names
-        super(GeneralizedPopen, self).__init__(*args, **kwargs)
-
-
-# shutil.which is not available on Python 2.7.
-def which(cmd, mode=os.F_OK | os.X_OK, path=None):
-    """Given a command, mode, and a PATH string, return the path which
-    conforms to the given mode on the PATH, or None if there is no such
-    file.
-
-    `mode` defaults to os.F_OK | os.X_OK. `path` defaults to the result
-    of os.environ.get("PATH"), or can be overridden with a custom search
-    path.
-
-    """
-    # Check that a given file can be accessed with the correct mode.
-    # Additionally check that `file` is not a directory, as on Windows
-    # directories pass the os.access check.
-    def _access_check(fn, mode):
-        return (os.path.exists(fn) and os.access(fn, mode)
-                and not os.path.isdir(fn))
-
-    # If we're given a path with a directory part, look it up directly rather
-    # than referring to PATH directories. This includes checking relative to the
-    # current directory, e.g. ./script
-    if os.path.dirname(cmd):
-        if _access_check(cmd, mode):
-            return cmd
-        return None
-
-    if path is None:
-        path = os.environ.get("PATH", os.defpath)
-    if not path:
-        return None
-    path = path.split(os.pathsep)
-
-    if sys.platform == "win32":
-        # The current directory takes precedence on Windows.
-        if os.curdir not in path:
-            path.insert(0, os.curdir)
-
-        # PATHEXT is necessary to check on Windows.
-        pathext = os.environ.get("PATHEXT", "").split(os.pathsep)
-        # See if the given file matches any of the expected path extensions.
-        # This will allow us to short circuit when given "python.exe".
-        # If it does match, only test that one, otherwise we have to try
-        # others.
-        if any(cmd.lower().endswith(ext.lower()) for ext in pathext):
-            files = [cmd]
-        else:
-            files = [cmd + ext for ext in pathext]
-    else:
-        # On other platforms you don't have things like PATHEXT to tell you
-        # what file suffixes are executable, so just pass on cmd as-is.
-        files = [cmd]
-
-    seen = set()
-    for dir in path:
-        normdir = os.path.normcase(dir)
-        if normdir not in seen:
-            seen.add(normdir)
-            for thefile in files:
-                name = os.path.join(dir, thefile)
-                if _access_check(name, mode):
-                    return name
-    return None
-
-
-if not is_py3:
-    # Simplified backport of Python 3 weakref.finalize:
-    # https://github.com/python/cpython/blob/ded4737989316653469763230036b04513cb62b3/Lib/weakref.py#L502-L662
-    class finalize(object):
-        """Class for finalization of weakrefable objects.
-
-        finalize(obj, func, *args, **kwargs) returns a callable finalizer
-        object which will be called when obj is garbage collected. The
-        first time the finalizer is called it evaluates func(*arg, **kwargs)
-        and returns the result. After this the finalizer is dead, and
-        calling it just returns None.
-
-        When the program exits any remaining finalizers will be run.
-        """
-
-        # Finalizer objects don't have any state of their own.
-        # This ensures that they cannot be part of a ref-cycle.
-        __slots__ = ()
-        _registry = {}
-
-        def __init__(self, obj, func, *args, **kwargs):
-            info = functools.partial(func, *args, **kwargs)
-            info.weakref = weakref.ref(obj, self)
-            self._registry[self] = info
-
-        # To me it's an absolute mystery why in Python 2 we need _=None. It
-        # makes really no sense since it's never really called. Then again it
-        # might be called by Python 2.7 itself, but weakref.finalize is not
-        # documented in Python 2 and therefore shouldn't be randomly called.
-        # We never call this stuff with a parameter and therefore this
-        # parameter should not be needed. But it is. ~dave
-        def __call__(self, _=None):
-            """Return func(*args, **kwargs) if alive."""
-            info = self._registry.pop(self, None)
-            if info:
-                return info()
-
-        @classmethod
-        def _exitfunc(cls):
-            if not cls._registry:
-                return
-            for finalizer in list(cls._registry):
-                try:
-                    finalizer()
-                except Exception:
-                    sys.excepthook(*sys.exc_info())
-                assert finalizer not in cls._registry
-
-    atexit.register(finalize._exitfunc)
-    weakref.finalize = finalize
-
-
-if is_py3 and sys.version_info[1] > 5:
-    from inspect import unwrap
-else:
-    # Only Python >=3.6 does properly limit the amount of unwraps. This is very
-    # relevant in the case of unittest.mock.patch.
-    # Below is the implementation of Python 3.7.
-    def unwrap(func, stop=None):
-        """Get the object wrapped by *func*.
-
-       Follows the chain of :attr:`__wrapped__` attributes returning the last
-       object in the chain.
-
-       *stop* is an optional callback accepting an object in the wrapper chain
-       as its sole argument that allows the unwrapping to be terminated early if
-       the callback returns a true value. If the callback never returns a true
-       value, the last object in the chain is returned as usual. For example,
-       :func:`signature` uses this to stop unwrapping if any object in the
-       chain has a ``__signature__`` attribute defined.
-
-       :exc:`ValueError` is raised if a cycle is encountered.
-
-        """
-        if stop is None:
-            def _is_wrapper(f):
-                return hasattr(f, '__wrapped__')
-        else:
-            def _is_wrapper(f):
-                return hasattr(f, '__wrapped__') and not stop(f)
-        f = func  # remember the original func for error reporting
-        # Memoise by id to tolerate non-hashable objects, but store objects to
-        # ensure they aren't destroyed, which would allow their IDs to be reused.
-        memo = {id(f): f}
-        recursion_limit = sys.getrecursionlimit()
-        while _is_wrapper(func):
-            func = func.__wrapped__
-            id_func = id(func)
-            if (id_func in memo) or (len(memo) >= recursion_limit):
-                raise ValueError('wrapper loop when unwrapping {!r}'.format(f))
-            memo[id_func] = func
-        return func

jedi/api/__init__.py

@@ -6,17 +6,13 @@ Additionally you can add a debug function with :func:`set_debug_function`.
 Alternatively, if you don't need a custom function and are happy with printing
 debug messages to stdout, simply call :func:`set_debug_function` without
 arguments.
-
-.. warning:: Please, note that Jedi is **not thread safe**.
 """
-import os
 import sys
-import warnings
+from pathlib import Path
 
 import parso
 from parso.python import tree
 
-from jedi._compatibility import force_unicode, cast_path, is_py3
 from jedi.parser_utils import get_executable_nodes
 from jedi import debug
 from jedi import settings
@@ -26,15 +22,18 @@ from jedi.api import classes
 from jedi.api import interpreter
 from jedi.api import helpers
 from jedi.api.helpers import validate_line_column
-from jedi.api.completion import Completion
+from jedi.api.completion import Completion, search_in_module
 from jedi.api.keywords import KeywordName
 from jedi.api.environment import InterpreterEnvironment
 from jedi.api.project import get_default_project, Project
+from jedi.api.errors import parso_to_jedi_errors
+from jedi.api import refactoring
+from jedi.api.refactoring.extract import extract_function, extract_variable
 from jedi.inference import InferenceState
 from jedi.inference import imports
 from jedi.inference.references import find_references
 from jedi.inference.arguments import try_iter_content
-from jedi.inference.helpers import get_module_names, infer_call_of_leaf
+from jedi.inference.helpers import infer_call_of_leaf
 from jedi.inference.sys_path import transform_path_to_dotted
 from jedi.inference.syntax_tree import tree_name_to_values
 from jedi.inference.value import ModuleValue
@@ -42,93 +41,96 @@ from jedi.inference.base_value import ValueSet
 from jedi.inference.value.iterable import unpack_tuple_to_dict
 from jedi.inference.gradual.conversion import convert_names, convert_values
 from jedi.inference.gradual.utils import load_proper_stub_module
+from jedi.inference.utils import to_list
 
 # Jedi uses lots and lots of recursion. By setting this a little bit higher, we
 # can remove some "maximum recursion depth" errors.
 sys.setrecursionlimit(3000)
 
 
-class Script(object):
+class Script:
     """
     A Script is the base for completions, goto or whatever you want to do with
-    |jedi|.
+    Jedi. The counter part of this class is :class:`Interpreter`, which works
+    with actual dictionaries and can work with a REPL. This class
+    should be used when a user edits code in an editor.
 
-    You can either use the ``source`` parameter or ``path`` to read a file.
+    You can either use the ``code`` parameter or ``path`` to read a file.
     Usually you're going to want to use both of them (in an editor).
 
-    The script might be analyzed in a different ``sys.path`` than |jedi|:
+    The Script's ``sys.path`` is very customizable:
 
-    - if `sys_path` parameter is not ``None``, it will be used as ``sys.path``
-      for the script;
+    - If `project` is provided with a ``sys_path``, that is going to be used.
+    - If `environment` is provided, its ``sys.path`` will be used
+      (see :func:`Environment.get_sys_path <jedi.api.environment.Environment.get_sys_path>`);
+    - Otherwise ``sys.path`` will match that of the default environment of
+      Jedi, which typically matches the sys path that was used at the time
+      when Jedi was imported.
 
-    - if `sys_path` parameter is ``None`` and ``VIRTUAL_ENV`` environment
-      variable is defined, ``sys.path`` for the specified environment will be
-      guessed (see :func:`jedi.inference.sys_path.get_venv_path`) and used for
-      the script;
+    Most methods have a ``line`` and a ``column`` parameter. Lines in Jedi are
+    always 1-based and columns are always zero based. To avoid repetition they
+    are not always documented. You can omit both line and column. Jedi will
+    then just do whatever action you are calling at the end of the file. If you
+    provide only the line, just will complete at the end of that line.
 
-    - otherwise ``sys.path`` will match that of |jedi|.
+    .. warning:: By default :attr:`jedi.settings.fast_parser` is enabled, which means
+        that parso reuses modules (i.e. they are not immutable). With this setting
+        Jedi is **not thread safe** and it is also not safe to use multiple
+        :class:`.Script` instances and its definitions at the same time.
 
-    :param source: The source code of the current file, separated by newlines.
-    :type source: str
-    :param line: Deprecated, please use it directly on e.g. `.complete`
-    :type line: int
-    :param column: Deprecated, please use it directly on e.g. `.complete`
-    :type column: int
+        If you are a normal plugin developer this should not be an issue. It is
+        an issue for people that do more complex stuff with Jedi.
+
+        This is purely a performance optimization and works pretty well for all
+        typical usages, however consider to turn the setting off if it causes
+        you problems. See also
+        `this discussion <https://github.com/davidhalter/jedi/issues/1240>`_.
+
+    :param code: The source code of the current file, separated by newlines.
+    :type code: str
     :param path: The path of the file in the file system, or ``''`` if
         it hasn't been saved yet.
-    :type path: str or None
-    :param encoding: The encoding of ``source``, if it is not a
-        ``unicode`` object (default ``'utf-8'``).
-    :type encoding: str
-    :param sys_path: ``sys.path`` to use during analysis of the script
-    :type sys_path: list
-    :param environment: TODO
-    :type environment: Environment
+    :type path: str or pathlib.Path or None
+    :param Environment environment: Provide a predefined :ref:`Environment <environments>`
+        to work with a specific Python version or virtualenv.
+    :param Project project: Provide a :class:`.Project` to make sure finding
+        references works well, because the right folder is searched. There are
+        also ways to modify the sys path and other things.
     """
-    def __init__(self, source=None, line=None, column=None, path=None,
-                 encoding='utf-8', sys_path=None, environment=None,
-                 _project=None):
+    def __init__(self, code=None, *, path=None, environment=None, project=None):
         self._orig_path = path
-        # An empty path (also empty string) should always result in no path.
-        self.path = os.path.abspath(path) if path else None
+        if isinstance(path, str):
+            path = Path(path)
+
+        self.path = path.absolute() if path else None
+
+        if code is None:
+            if path is None:
+                raise ValueError("Must provide at least one of code or path")
 
-        if source is None:
             # TODO add a better warning than the traceback!
             with open(path, 'rb') as f:
-                source = f.read()
+                code = f.read()
 
-        # Load the Python grammar of the current interpreter.
-        self._grammar = parso.load_grammar()
-
-        if sys_path is not None and not is_py3:
-            sys_path = list(map(force_unicode, sys_path))
-
-        project = _project
         if project is None:
             # Load the Python grammar of the current interpreter.
-            project = get_default_project(
-                os.path.dirname(self.path)if path else os.getcwd()
-            )
-        # TODO deprecate and remove sys_path from the Script API.
-        if sys_path is not None:
-            project._sys_path = sys_path
+            project = get_default_project(None if self.path is None else self.path.parent)
+
         self._inference_state = InferenceState(
             project, environment=environment, script_path=self.path
         )
         debug.speed('init')
-        self._module_node, source = self._inference_state.parse_and_get_code(
-            code=source,
+        self._module_node, code = self._inference_state.parse_and_get_code(
+            code=code,
             path=self.path,
-            encoding=encoding,
-            use_latest_grammar=path and path.endswith('.pyi'),
+            use_latest_grammar=path and path.suffix == '.pyi',
             cache=False,  # No disk cache, because the current script often changes.
             diff_cache=settings.fast_parser,
             cache_path=settings.cache_directory,
         )
         debug.speed('parsed')
-        self._code_lines = parso.split_lines(source, keepends=True)
-        self._code = source
-        self._pos = line, column
+        self._code_lines = parso.split_lines(code, keepends=True)
+        self._code = code
 
         cache.clear_time_caches()
         debug.reset_time()
@@ -151,11 +153,12 @@ class Script(object):
         if self.path is None:
             file_io = None
         else:
-            file_io = KnownContentFileIO(cast_path(self.path), self._code)
-        if self.path is not None and self.path.endswith('.pyi'):
+            file_io = KnownContentFileIO(self.path, self._code)
+        if self.path is not None and self.path.suffix == '.pyi':
             # We are in a stub file. Try to load the stub properly.
             stub_module = load_proper_stub_module(
                 self._inference_state,
+                self._inference_state.latest_grammar,
                 file_io,
                 names,
                 self._module_node
@@ -173,7 +176,7 @@ class Script(object):
             code_lines=self._code_lines,
             is_package=is_package,
         )
-        if names[0] not in ('builtins', '__builtin__', 'typing'):
+        if names[0] not in ('builtins', 'typing'):
             # These modules are essential for Jedi, so don't overwrite them.
             self._inference_state.module_cache.add(names, ValueSet([module]))
         return module
@@ -189,19 +192,21 @@ class Script(object):
         )
 
     @validate_line_column
-    def complete(self, line=None, column=None, **kwargs):
+    def complete(self, line=None, column=None, *, fuzzy=False):
         """
-        Return :class:`classes.Completion` objects. Those objects contain
-        information about the completions, more than just names.
+        Completes objects under the cursor.
+
+        Those objects contain information about the completions, more than just
+        names.
 
         :param fuzzy: Default False. Will return fuzzy completions, which means
             that e.g. ``ooa`` will match ``foobar``.
-        :return: Completion objects, sorted by name and ``__`` comes last.
-        :rtype: list of :class:`classes.Completion`
+        :return: Completion objects, sorted by name. Normal names appear
+            before "private" names that start with ``_`` and those appear
+            before magic methods and name mangled names that start with ``__``.
+        :rtype: list of :class:`.Completion`
         """
-        return self._complete(line, column, **kwargs)
-
-    def _complete(self, line, column, fuzzy=False):  # Python 2...
+        self._inference_state.reset_recursion_limitations()
         with debug.increase_indent_cm('complete'):
             completion = Completion(
                 self._inference_state, self._get_module_context(), self._code_lines,
@@ -209,40 +214,35 @@ class Script(object):
             )
             return completion.complete()
 
-    def completions(self, fuzzy=False):
-        # Deprecated, will be removed.
-        return self.complete(*self._pos, fuzzy=fuzzy)
-
     @validate_line_column
-    def infer(self, line=None, column=None, **kwargs):
+    def infer(self, line=None, column=None, *, only_stubs=False, prefer_stubs=False):
         """
-        Return the definitions of a the path under the cursor.  goto function!
-        This follows complicated paths and returns the end, not the first
-        definition. The big difference between :meth:`goto` and
+        Return the definitions of under the cursor. It is basically a wrapper
+        around Jedi's type inference.
+
+        This method follows complicated paths and returns the end, not the
+        first definition. The big difference between :meth:`goto` and
         :meth:`infer` is that :meth:`goto` doesn't
         follow imports and statements. Multiple objects may be returned,
-        because Python itself is a dynamic language, which means depending on
-        an option you can have two different versions of a function.
+        because depending on an option you can have two different versions of a
+        function.
 
-        :param only_stubs: Only return stubs for this goto call.
-        :param prefer_stubs: Prefer stubs to Python objects for this type
-            inference call.
-        :rtype: list of :class:`classes.Definition`
+        :param only_stubs: Only return stubs for this method.
+        :param prefer_stubs: Prefer stubs to Python objects for this method.
+        :rtype: list of :class:`.Name`
         """
-        with debug.increase_indent_cm('infer'):
-            return self._infer(line, column, **kwargs)
-
-    def goto_definitions(self, **kwargs):
-        # Deprecated, will be removed.
-        return self.infer(*self._pos, **kwargs)
-
-    def _infer(self, line, column, only_stubs=False, prefer_stubs=False):
+        self._inference_state.reset_recursion_limitations()
         pos = line, column
         leaf = self._module_node.get_name_of_position(pos)
         if leaf is None:
             leaf = self._module_node.get_leaf_for_position(pos)
             if leaf is None or leaf.type == 'string':
                 return []
+            if leaf.end_pos == (line, column) and leaf.type == 'operator':
+                next_ = leaf.get_next_leaf()
+                if next_.start_pos == leaf.end_pos \
+                        and next_.type in ('number', 'string', 'keyword'):
+                    leaf = next_
 
         context = self._get_module_context().create_context(leaf)
 
@@ -253,39 +253,29 @@ class Script(object):
             prefer_stubs=prefer_stubs,
         )
 
-        defs = [classes.Definition(self._inference_state, c.name) for c in values]
+        defs = [classes.Name(self._inference_state, c.name) for c in values]
         # The additional set here allows the definitions to become unique in an
         # API sense. In the internals we want to separate more things than in
         # the API.
         return helpers.sorted_definitions(set(defs))
 
-    def goto_assignments(self, follow_imports=False, follow_builtin_imports=False, **kwargs):
-        # Deprecated, will be removed.
-        return self.goto(*self._pos,
-                         follow_imports=follow_imports,
-                         follow_builtin_imports=follow_builtin_imports,
-                         **kwargs)
-
     @validate_line_column
-    def goto(self, line=None, column=None, **kwargs):
+    def goto(self, line=None, column=None, *, follow_imports=False, follow_builtin_imports=False,
+             only_stubs=False, prefer_stubs=False):
         """
-        Return the first definition found, while optionally following imports.
-        Multiple objects may be returned, because Python itself is a
-        dynamic language, which means depending on an option you can have two
+        Goes to the name that defined the object under the cursor. Optionally
+        you can follow imports.
+        Multiple objects may be returned, depending on an if you can have two
         different versions of a function.
 
-        :param follow_imports: The goto call will follow imports.
-        :param follow_builtin_imports: If follow_imports is True will decide if
-            it follow builtin imports.
-        :param only_stubs: Only return stubs for this goto call.
-        :param prefer_stubs: Prefer stubs to Python objects for this goto call.
-        :rtype: list of :class:`classes.Definition`
+        :param follow_imports: The method will follow imports.
+        :param follow_builtin_imports: If ``follow_imports`` is True will try
+            to look up names in builtins (i.e. compiled or extension modules).
+        :param only_stubs: Only return stubs for this method.
+        :param prefer_stubs: Prefer stubs to Python objects for this method.
+        :rtype: list of :class:`.Name`
         """
-        with debug.increase_indent_cm('goto'):
-            return self._goto(line, column, **kwargs)
-
-    def _goto(self, line, column, follow_imports=False, follow_builtin_imports=False,
-              only_stubs=False, prefer_stubs=False):
+        self._inference_state.reset_recursion_limitations()
         tree_name = self._module_node.get_name_of_position((line, column))
         if tree_name is None:
             # Without a name we really just want to jump to the result e.g.
@@ -310,78 +300,138 @@ class Script(object):
             names = list(name.goto())
 
         if follow_imports:
-            names = helpers.filter_follow_imports(names)
+            names = helpers.filter_follow_imports(names, follow_builtin_imports)
         names = convert_names(
             names,
             only_stubs=only_stubs,
             prefer_stubs=prefer_stubs,
         )
 
-        defs = [classes.Definition(self._inference_state, d) for d in set(names)]
-        return helpers.sorted_definitions(defs)
+        defs = [classes.Name(self._inference_state, d) for d in set(names)]
+        # Avoid duplicates
+        return list(set(helpers.sorted_definitions(defs)))
+
+    def search(self, string, *, all_scopes=False):
+        """
+        Searches a name in the current file. For a description of how the
+        search string should look like, please have a look at
+        :meth:`.Project.search`.
+
+        :param bool all_scopes: Default False; searches not only for
+            definitions on the top level of a module level, but also in
+            functions and classes.
+        :yields: :class:`.Name`
+        """
+        return self._search_func(string, all_scopes=all_scopes)
+
+    @to_list
+    def _search_func(self, string, all_scopes=False, complete=False, fuzzy=False):
+        names = self._names(all_scopes=all_scopes)
+        wanted_type, wanted_names = helpers.split_search_string(string)
+        return search_in_module(
+            self._inference_state,
+            self._get_module_context(),
+            names=names,
+            wanted_type=wanted_type,
+            wanted_names=wanted_names,
+            complete=complete,
+            fuzzy=fuzzy,
+        )
+
+    def complete_search(self, string, **kwargs):
+        """
+        Like :meth:`.Script.search`, but completes that string. If you want to
+        have all possible definitions in a file you can also provide an empty
+        string.
+
+        :param bool all_scopes: Default False; searches not only for
+            definitions on the top level of a module level, but also in
+            functions and classes.
+        :param fuzzy: Default False. Will return fuzzy completions, which means
+            that e.g. ``ooa`` will match ``foobar``.
+        :yields: :class:`.Completion`
+        """
+        return self._search_func(string, complete=True, **kwargs)
 
     @validate_line_column
     def help(self, line=None, column=None):
         """
-        Works like goto and returns a list of Definition objects. Returns
-        additional definitions for keywords and operators.
+        Used to display a help window to users.  Uses :meth:`.Script.goto` and
+        returns additional definitions for keywords and operators.
 
-        The additional definitions are of ``Definition(...).type == 'keyword'``.
+        Typically you will want to display :meth:`.BaseName.docstring` to the
+        user for all the returned definitions.
+
+        The additional definitions are ``Name(...).type == 'keyword'``.
         These definitions do not have a lot of value apart from their docstring
-        attribute, which contains the output of Python's ``help()`` function.
+        attribute, which contains the output of Python's :func:`help` function.
 
-        :rtype: list of :class:`classes.Definition`
+        :rtype: list of :class:`.Name`
         """
+        self._inference_state.reset_recursion_limitations()
         definitions = self.goto(line, column, follow_imports=True)
         if definitions:
             return definitions
         leaf = self._module_node.get_leaf_for_position((line, column))
-        if leaf.type in ('keyword', 'operator', 'error_leaf'):
-            reserved = self._grammar._pgen_grammar.reserved_syntax_strings.keys()
-            if leaf.value in reserved:
-                name = KeywordName(self._inference_state, leaf.value)
-                return [classes.Definition(self._inference_state, name)]
-        return []
 
-    def usages(self, **kwargs):
-        # Deprecated, will be removed.
-        return self.get_references(*self._pos, **kwargs)
+        if leaf is not None and leaf.end_pos == (line, column) and leaf.type == 'newline':
+            next_ = leaf.get_next_leaf()
+            if next_ is not None and next_.start_pos == leaf.end_pos:
+                leaf = next_
+
+        if leaf is not None and leaf.type in ('keyword', 'operator', 'error_leaf'):
+            def need_pydoc():
+                if leaf.value in ('(', ')', '[', ']'):
+                    if leaf.parent.type == 'trailer':
+                        return False
+                    if leaf.parent.type == 'atom':
+                        return False
+                grammar = self._inference_state.grammar
+                # This parso stuff is not public, but since I control it, this
+                # is fine :-) ~dave
+                reserved = grammar._pgen_grammar.reserved_syntax_strings.keys()
+                return leaf.value in reserved
+
+            if need_pydoc():
+                name = KeywordName(self._inference_state, leaf.value)
+                return [classes.Name(self._inference_state, name)]
+        return []
 
     @validate_line_column
     def get_references(self, line=None, column=None, **kwargs):
         """
-        Return :class:`classes.Definition` objects, which contain all
-        names that point to the definition of the name under the cursor. This
-        is very useful for refactoring (renaming), or to show all references of
-        a variable.
+        Lists all references of a variable in a project. Since this can be
+        quite hard to do for Jedi, if it is too complicated, Jedi will stop
+        searching.
 
-        :param include_builtins: Default True, checks if a reference is a
-            builtin (e.g. ``sys``) and in that case does not return it.
-        :rtype: list of :class:`classes.Definition`
+        :param include_builtins: Default ``True``. If ``False``, checks if a definition
+            is a builtin (e.g. ``sys``) and in that case does not return it.
+        :param scope: Default ``'project'``. If ``'file'``, include references in
+            the current module only.
+        :rtype: list of :class:`.Name`
         """
+        self._inference_state.reset_recursion_limitations()
 
-        def _references(include_builtins=True):
+        def _references(include_builtins=True, scope='project'):
+            if scope not in ('project', 'file'):
+                raise ValueError('Only the scopes "file" and "project" are allowed')
             tree_name = self._module_node.get_name_of_position((line, column))
             if tree_name is None:
                 # Must be syntax
                 return []
 
-            names = find_references(self._get_module_context(), tree_name)
+            names = find_references(self._get_module_context(), tree_name, scope == 'file')
 
-            definitions = [classes.Definition(self._inference_state, n) for n in names]
-            if not include_builtins:
+            definitions = [classes.Name(self._inference_state, n) for n in names]
+            if not include_builtins or scope == 'file':
                 definitions = [d for d in definitions if not d.in_builtin_module()]
             return helpers.sorted_definitions(definitions)
         return _references(**kwargs)
 
-    def call_signatures(self):
-        # Deprecated, will be removed.
-        return self.get_signatures(*self._pos)
-
     @validate_line_column
     def get_signatures(self, line=None, column=None):
         """
-        Return the function object of the call you're currently in.
+        Return the function object of the call under the cursor.
 
         E.g. if the cursor is here::
 
@@ -393,8 +443,9 @@ class Script(object):
 
         This would return an empty list..
 
-        :rtype: list of :class:`classes.Signature`
+        :rtype: list of :class:`.Signature`
         """
+        self._inference_state.reset_recursion_limitations()
         pos = line, column
         call_details = helpers.get_signature_details(self._module_node, pos)
         if call_details is None:
@@ -417,6 +468,12 @@ class Script(object):
 
     @validate_line_column
     def get_context(self, line=None, column=None):
+        """
+        Returns the scope context under the cursor. This basically means the
+        function, class or module where the cursor is at.
+
+        :rtype: :class:`.Name`
+        """
         pos = (line, column)
         leaf = self._module_node.get_leaf_for_position(pos, include_prefixes=True)
         if leaf.start_pos > pos or leaf.type == 'endmarker':
@@ -439,7 +496,7 @@ class Script(object):
         while context.name is None:
             context = context.parent_context  # comprehensions
 
-        definition = classes.Definition(self._inference_state, context.name)
+        definition = classes.Name(self._inference_state, context.name)
         while definition.type != 'module':
             name = definition._name  # TODO private access
             tree_name = name.tree_name
@@ -486,66 +543,186 @@ class Script(object):
 
     def get_names(self, **kwargs):
         """
-        Returns a list of `Definition` objects, containing name parts.
-        This means you can call ``Definition.goto()`` and get the
-        reference of a name.
+        Returns names defined in the current file.
 
-        :param all_scopes: If True lists the names of all scopes instead of only
-            the module namespace.
+        :param all_scopes: If True lists the names of all scopes instead of
+            only the module namespace.
         :param definitions: If True lists the names that have been defined by a
             class, function or a statement (``a = b`` returns ``a``).
         :param references: If True lists all the names that are not listed by
             ``definitions=True``. E.g. ``a = b`` returns ``b``.
+        :rtype: list of :class:`.Name`
         """
-        return self._names(**kwargs)  # Python 2...
+        names = self._names(**kwargs)
+        return [classes.Name(self._inference_state, n) for n in names]
+
+    def get_syntax_errors(self):
+        """
+        Lists all syntax errors in the current file.
+
+        :rtype: list of :class:`.SyntaxError`
+        """
+        return parso_to_jedi_errors(self._inference_state.grammar, self._module_node)
 
     def _names(self, all_scopes=False, definitions=True, references=False):
-        def def_ref_filter(_def):
-            is_def = _def._name.tree_name.is_definition()
-            return definitions and is_def or references and not is_def
-
+        self._inference_state.reset_recursion_limitations()
         # Set line/column to a random position, because they don't matter.
         module_context = self._get_module_context()
         defs = [
-            classes.Definition(
-                self._inference_state,
-                module_context.create_name(name)
-            ) for name in get_module_names(self._module_node, all_scopes)
+            module_context.create_name(name)
+            for name in helpers.get_module_names(
+                self._module_node,
+                all_scopes=all_scopes,
+                definitions=definitions,
+                references=references,
+            )
         ]
-        return sorted(filter(def_ref_filter, defs), key=lambda x: (x.line, x.column))
+        return sorted(defs, key=lambda x: x.start_pos)
+
+    def rename(self, line=None, column=None, *, new_name):
+        """
+        Renames all references of the variable under the cursor.
+
+        :param new_name: The variable under the cursor will be renamed to this
+            string.
+        :raises: :exc:`.RefactoringError`
+        :rtype: :class:`.Refactoring`
+        """
+        definitions = self.get_references(line, column, include_builtins=False)
+        return refactoring.rename(self._inference_state, definitions, new_name)
+
+    @validate_line_column
+    def extract_variable(self, line, column, *, new_name, until_line=None, until_column=None):
+        """
+        Moves an expression to a new statement.
+
+        For example if you have the cursor on ``foo`` and provide a
+        ``new_name`` called ``bar``::
+
+            foo = 3.1
+            x = int(foo + 1)
+
+        the code above will become::
+
+            foo = 3.1
+            bar = foo + 1
+            x = int(bar)
+
+        :param new_name: The expression under the cursor will be renamed to
+            this string.
+        :param int until_line: The the selection range ends at this line, when
+            omitted, Jedi will be clever and try to define the range itself.
+        :param int until_column: The the selection range ends at this column, when
+            omitted, Jedi will be clever and try to define the range itself.
+        :raises: :exc:`.RefactoringError`
+        :rtype: :class:`.Refactoring`
+        """
+        if until_line is None and until_column is None:
+            until_pos = None
+        else:
+            if until_line is None:
+                until_line = line
+            if until_column is None:
+                until_column = len(self._code_lines[until_line - 1])
+            until_pos = until_line, until_column
+        return extract_variable(
+            self._inference_state, self.path, self._module_node,
+            new_name, (line, column), until_pos
+        )
+
+    @validate_line_column
+    def extract_function(self, line, column, *, new_name, until_line=None, until_column=None):
+        """
+        Moves an expression to a new function.
+
+        For example if you have the cursor on ``foo`` and provide a
+        ``new_name`` called ``bar``::
+
+            global_var = 3
+
+            def x():
+                foo = 3.1
+                x = int(foo + 1 + global_var)
+
+        the code above will become::
+
+            global_var = 3
+
+            def bar(foo):
+                return int(foo + 1 + global_var)
+
+            def x():
+                foo = 3.1
+                x = bar(foo)
+
+        :param new_name: The expression under the cursor will be replaced with
+            a function with this name.
+        :param int until_line: The the selection range ends at this line, when
+            omitted, Jedi will be clever and try to define the range itself.
+        :param int until_column: The the selection range ends at this column, when
+            omitted, Jedi will be clever and try to define the range itself.
+        :raises: :exc:`.RefactoringError`
+        :rtype: :class:`.Refactoring`
+        """
+        if until_line is None and until_column is None:
+            until_pos = None
+        else:
+            if until_line is None:
+                until_line = line
+            if until_column is None:
+                until_column = len(self._code_lines[until_line - 1])
+            until_pos = until_line, until_column
+        return extract_function(
+            self._inference_state, self.path, self._get_module_context(),
+            new_name, (line, column), until_pos
+        )
+
+    def inline(self, line=None, column=None):
+        """
+        Inlines a variable under the cursor. This is basically the opposite of
+        extracting a variable. For example with the cursor on bar::
+
+            foo = 3.1
+            bar = foo + 1
+            x = int(bar)
+
+        the code above will become::
+
+            foo = 3.1
+            x = int(foo + 1)
+
+        :raises: :exc:`.RefactoringError`
+        :rtype: :class:`.Refactoring`
+        """
+        names = [d._name for d in self.get_references(line, column, include_builtins=True)]
+        return refactoring.inline(self._inference_state, names)
 
 
 class Interpreter(Script):
     """
-    Jedi API for Python REPLs.
+    Jedi's API for Python REPLs.
 
-    In addition to completion of simple attribute access, Jedi
-    supports code completion based on static code analysis.
-    Jedi can complete attributes of object which is not initialized
-    yet.
+    Implements all of the methods that are present in :class:`.Script` as well.
+
+    In addition to completions that normal REPL completion does like
+    ``str.upper``, Jedi also supports code completion based on static code
+    analysis. For example Jedi will complete ``str().upper``.
 
     >>> from os.path import join
     >>> namespace = locals()
     >>> script = Interpreter('join("").up', [namespace])
     >>> print(script.complete()[0].name)
     upper
+
+    All keyword arguments are same as the arguments for :class:`.Script`.
+
+    :param str code: Code to parse.
+    :type namespaces: typing.List[dict]
+    :param namespaces: A list of namespace dictionaries such as the one
+        returned by :func:`globals` and :func:`locals`.
     """
-    _allow_descriptor_getattr_default = True
 
-    def __init__(self, source, namespaces, **kwds):
-        """
-        Parse `source` and mixin interpreted Python objects from `namespaces`.
-
-        :type source: str
-        :arg  source: Code to parse.
-        :type namespaces: list of dict
-        :arg  namespaces: a list of namespace dictionaries such as the one
-                          returned by :func:`locals`.
-
-        Other optional arguments are same as the ones for :class:`Script`.
-        If `line` and `column` are None, they are assumed be at the end of
-        `source`.
-        """
+    def __init__(self, code, namespaces, *, project=None, **kwds):
         try:
             namespaces = [dict(n) for n in namespaces]
         except Exception:
@@ -558,16 +735,32 @@ class Interpreter(Script):
             if not isinstance(environment, InterpreterEnvironment):
                 raise TypeError("The environment needs to be an InterpreterEnvironment subclass.")
 
-        super(Interpreter, self).__init__(source, environment=environment,
-                                          _project=Project(os.getcwd()), **kwds)
+        if project is None:
+            project = Project(Path.cwd())
+
+        super().__init__(code, environment=environment, project=project, **kwds)
+
         self.namespaces = namespaces
-        self._inference_state.allow_descriptor_getattr = self._allow_descriptor_getattr_default
+        self._inference_state.allow_unsafe_executions = \
+            settings.allow_unsafe_interpreter_executions
+        # Dynamic params search is important when we work on functions that are
+        # called by other pieces of code. However for interpreter completions
+        # this is not important at all, because the current code is always new
+        # and will never be called by something.
+        # Also sometimes this logic goes a bit too far like in
+        # https://github.com/ipython/ipython/issues/13866, where it takes
+        # seconds to do a simple completion.
+        self._inference_state.do_dynamic_params_search = False
 
     @cache.memoize_method
     def _get_module_context(self):
+        if self.path is None:
+            file_io = None
+        else:
+            file_io = KnownContentFileIO(self.path, self._code)
         tree_module_value = ModuleValue(
             self._inference_state, self._module_node,
-            file_io=KnownContentFileIO(self.path, self._code),
+            file_io=file_io,
             string_names=('__main__',),
             code_lines=self._code_lines,
         )
@@ -577,31 +770,17 @@ class Interpreter(Script):
         )
 
 
-def names(source=None, path=None, encoding='utf-8', all_scopes=False,
-          definitions=True, references=False, environment=None):
-    warnings.warn(
-        "Deprecated since version 0.16.0. Use Script(...).get_names instead.",
-        DeprecationWarning,
-        stacklevel=2
-    )
-
-    return Script(source, path=path, encoding=encoding).get_names(
-        all_scopes=all_scopes,
-        definitions=definitions,
-        references=references,
-    )
-
-
 def preload_module(*modules):
     """
     Preloading modules tells Jedi to load a module now, instead of lazy parsing
-    of modules. Usful for IDEs, to control which modules to load on startup.
+    of modules. This can be useful for IDEs, to control which modules to load
+    on startup.
 
     :param modules: different module names, list of string.
     """
     for m in modules:
         s = "import %s as x; x." % m
-        Script(s, path=None).complete(1, len(s))
+        Script(s).complete(1, len(s))
 
 
 def set_debug_function(func_cb=debug.print_to_stdout, warnings=True,
@@ -611,7 +790,7 @@ def set_debug_function(func_cb=debug.print_to_stdout, warnings=True,
 
     If you don't specify any arguments, debug messages will be printed to stdout.
 
-    :param func_cb: The callback function for debug messages, with n params.
+    :param func_cb: The callback function for debug messages.
     """
     debug.debug_function = func_cb
     debug.enable_warning = warnings

jedi/api/classes.py

@@ -1,23 +1,33 @@
 """
-The :mod:`jedi.api.classes` module contains the return classes of the API.
-These classes are the much bigger part of the whole API, because they contain
-the interesting information about completion and goto operations.
+There are a couple of classes documented in here:
+
+- :class:`.BaseName` as an abstact base class for almost everything.
+- :class:`.Name` used in a lot of places
+- :class:`.Completion` for completions
+- :class:`.BaseSignature` as a base class for signatures
+- :class:`.Signature` for :meth:`.Script.get_signatures` only
+- :class:`.ParamName` used for parameters of signatures
+- :class:`.Refactoring` for refactorings
+- :class:`.SyntaxError` for :meth:`.Script.get_syntax_errors` only
+
+These classes are the much biggest part of the API, because they contain
+the interesting information about all operations.
 """
 import re
-import sys
-import warnings
+from pathlib import Path
+from typing import Optional
 
-from parso.python.tree import search_ancestor
+from parso.tree import search_ancestor
 
 from jedi import settings
 from jedi import debug
 from jedi.inference.utils import unite
 from jedi.cache import memoize_method
-from jedi.inference import imports
-from jedi.inference.imports import ImportName
-from jedi.inference.gradual.typeshed import StubModuleValue
+from jedi.inference.compiled.mixed import MixedName
+from jedi.inference.names import ImportName, SubModuleName
+from jedi.inference.gradual.stub_value import StubModuleValue
 from jedi.inference.gradual.conversion import convert_names, convert_values
-from jedi.inference.base_value import ValueSet
+from jedi.inference.base_value import ValueSet, HasNoContext
 from jedi.api.keywords import KeywordName
 from jedi.api import completion_cache
 from jedi.api.helpers import filter_follow_imports
@@ -27,23 +37,30 @@ def _sort_names_by_start_pos(names):
     return sorted(names, key=lambda s: s.start_pos or (0, 0))
 
 
-def defined_names(inference_state, context):
+def defined_names(inference_state, value):
     """
     List sub-definitions (e.g., methods in class).
 
     :type scope: Scope
-    :rtype: list of Definition
+    :rtype: list of Name
     """
+    try:
+        context = value.as_context()
+    except HasNoContext:
+        return []
     filter = next(context.get_filters())
     names = [name for name in filter.values()]
-    return [Definition(inference_state, n) for n in _sort_names_by_start_pos(names)]
+    return [Name(inference_state, n) for n in _sort_names_by_start_pos(names)]
 
 
 def _values_to_definitions(values):
-    return [Definition(c.inference_state, c.name) for c in values]
+    return [Name(c.inference_state, c.name) for c in values]
 
 
-class BaseDefinition(object):
+class BaseName:
+    """
+    The base class for all definitions, completions and signatures.
+    """
     _mapping = {
         'posixpath': 'os.path',
         'riscospath': 'os.path',
@@ -57,7 +74,6 @@ class BaseDefinition(object):
         '_collections': 'collections',
         '_socket': 'socket',
         '_sqlite3': 'sqlite3',
-        '__builtin__': 'builtins',
     }
 
     _tuple_mapping = dict((tuple(k.split('.')), v) for (k, v) in {
@@ -80,13 +96,16 @@ class BaseDefinition(object):
         return self._name.get_root_context()
 
     @property
-    def module_path(self):
-        """Shows the file path of a module. e.g. ``/usr/lib/python2.7/os.py``"""
+    def module_path(self) -> Optional[Path]:
+        """
+        Shows the file path of a module. e.g. ``/usr/lib/python3.9/os.py``
+        """
         module = self._get_module_context()
         if module.is_stub() or not module.is_compiled():
             # Compiled modules should not return a module path even if they
             # have one.
-            return self._get_module_context().py__file__()
+            path: Optional[Path] = self._get_module_context().py__file__()
+            return path
 
         return None
 
@@ -111,7 +130,6 @@ class BaseDefinition(object):
         to Jedi, :meth:`jedi.Script.infer` should return a list of
         definition for ``sys``, ``f``, ``C`` and ``x``.
 
-        >>> from jedi._compatibility import no_unicode_pprint
         >>> from jedi import Script
         >>> source = '''
         ... import keyword
@@ -137,15 +155,15 @@ class BaseDefinition(object):
         so that it is easy to relate the result to the source code.
 
         >>> defs = sorted(defs, key=lambda d: d.line)
-        >>> no_unicode_pprint(defs)  # doctest: +NORMALIZE_WHITESPACE
-        [<Definition full_name='keyword', description='module keyword'>,
-         <Definition full_name='__main__.C', description='class C'>,
-         <Definition full_name='__main__.D', description='instance D'>,
-         <Definition full_name='__main__.f', description='def f'>]
+        >>> print(defs)  # doctest: +NORMALIZE_WHITESPACE
+        [<Name full_name='keyword', description='module keyword'>,
+         <Name full_name='__main__.C', description='class C'>,
+         <Name full_name='__main__.D', description='instance D'>,
+         <Name full_name='__main__.f', description='def f'>]
 
         Finally, here is what you can get from :attr:`type`:
 
-        >>> defs = [str(d.type) for d in defs]  # It's unicode and in Py2 has u before it.
+        >>> defs = [d.type for d in defs]
         >>> defs[0]
         'module'
         >>> defs[1]
@@ -155,8 +173,8 @@ class BaseDefinition(object):
         >>> defs[3]
         'function'
 
-        Valid values for are ``module``, ``class``, ``instance``, ``function``,
-        ``param``, ``path`` and ``keyword``.
+        Valid values for type are ``module``, ``class``, ``instance``, ``function``,
+        ``param``, ``path``, ``keyword``, ``property`` and ``statement``.
 
         """
         tree_name = self._name.tree_name
@@ -168,7 +186,7 @@ class BaseDefinition(object):
                     tree_name.is_definition():
                 resolve = True
 
-        if isinstance(self._name, imports.SubModuleName) or resolve:
+        if isinstance(self._name, SubModuleName) or resolve:
             for value in self._name.infer():
                 return value.api_type
         return self._name.api_type
@@ -176,7 +194,8 @@ class BaseDefinition(object):
     @property
     def module_name(self):
         """
-        The module name.
+        The module name, a bit similar to what ``__name__`` is in a random
+        Python module.
 
         >>> from jedi import Script
         >>> source = 'import json'
@@ -188,7 +207,9 @@ class BaseDefinition(object):
         return self._get_module_context().py__name__()
 
     def in_builtin_module(self):
-        """Whether this is a builtin module."""
+        """
+        Returns True, if this is a builtin module.
+        """
         value = self._get_module_context().get_value()
         if isinstance(value, StubModuleValue):
             return any(v.is_compiled() for v in value.non_stub_value_set)
@@ -210,6 +231,39 @@ class BaseDefinition(object):
             return None
         return start_pos[1]
 
+    def get_definition_start_position(self):
+        """
+        The (row, column) of the start of the definition range. Rows start with
+        1, columns start with 0.
+
+        :rtype: Optional[Tuple[int, int]]
+        """
+        if self._name.tree_name is None:
+            return None
+        definition = self._name.tree_name.get_definition()
+        if definition is None:
+            return self._name.start_pos
+        return definition.start_pos
+
+    def get_definition_end_position(self):
+        """
+        The (row, column) of the end of the definition range. Rows start with
+        1, columns start with 0.
+
+        :rtype: Optional[Tuple[int, int]]
+        """
+        if self._name.tree_name is None:
+            return None
+        definition = self._name.tree_name.get_definition()
+        if definition is None:
+            return self._name.tree_name.end_pos
+        if self.type in ("function", "class"):
+            last_leaf = definition.get_last_leaf()
+            if last_leaf.type == "newline":
+                return last_leaf.get_previous_leaf().end_pos
+            return last_leaf.end_pos
+        return definition.end_pos
+
     def docstring(self, raw=False, fast=True):
         r"""
         Return a document string for this completion object.
@@ -229,8 +283,8 @@ class BaseDefinition(object):
         Document for function f.
 
         Notice that useful extra information is added to the actual
-        docstring.  For function, it is signature.  If you need
-        actual docstring, use ``raw=True`` instead.
+        docstring, e.g. function signatures are prepended to their docstrings.
+        If you need the actual docstring, use ``raw=True`` instead.
 
         >>> print(script.infer(1, len('def f'))[0].docstring(raw=True))
         Document for function f.
@@ -265,12 +319,11 @@ class BaseDefinition(object):
     @property
     def description(self):
         """
-        A description of the :class:`.Definition` object, which is heavily used
+        A description of the :class:`.Name` object, which is heavily used
         in testing. e.g. for ``isinstance`` it returns ``def isinstance``.
 
         Example:
 
-        >>> from jedi._compatibility import no_unicode_pprint
         >>> from jedi import Script
         >>> source = '''
         ... def f():
@@ -283,10 +336,10 @@ class BaseDefinition(object):
         >>> script = Script(source)  # line is maximum by default
         >>> defs = script.infer(column=3)
         >>> defs = sorted(defs, key=lambda d: d.line)
-        >>> no_unicode_pprint(defs)  # doctest: +NORMALIZE_WHITESPACE
-        [<Definition full_name='__main__.f', description='def f'>,
-         <Definition full_name='__main__.C', description='class C'>]
-        >>> str(defs[0].description)  # strip literals in python2
+        >>> print(defs)  # doctest: +NORMALIZE_WHITESPACE
+        [<Name full_name='__main__.f', description='def f'>,
+         <Name full_name='__main__.C', description='class C'>]
+        >>> str(defs[0].description)
         'def f'
         >>> str(defs[1].description)
         'class C'
@@ -341,7 +394,7 @@ class BaseDefinition(object):
 
         names = self._name.get_qualified_names(include_module_names=True)
         if names is None:
-            return names
+            return None
 
         names = list(names)
         try:
@@ -352,26 +405,40 @@ class BaseDefinition(object):
         return '.'.join(names)
 
     def is_stub(self):
+        """
+        Returns True if the current name is defined in a stub file.
+        """
         if not self._name.is_value_name:
             return False
 
         return self._name.get_root_context().is_stub()
 
-    def goto(self, **kwargs):
-        with debug.increase_indent_cm('goto for %s' % self._name):
-            return self._goto(**kwargs)
+    def is_side_effect(self):
+        """
+        Checks if a name is defined as ``self.foo = 3``. In case of self, this
+        function would return False, for foo it would return True.
+        """
+        tree_name = self._name.tree_name
+        if tree_name is None:
+            return False
+        return tree_name.is_definition() and tree_name.parent.type == 'trailer'
 
-    def goto_assignments(self, **kwargs):  # Python 2...
-        warnings.warn(
-            "Deprecated since version 0.16.0. Use .goto.",
-            DeprecationWarning,
-            stacklevel=2
-        )
-        return self.goto(**kwargs)
+    @debug.increase_indent_cm('goto on name')
+    def goto(self, *, follow_imports=False, follow_builtin_imports=False,
+             only_stubs=False, prefer_stubs=False):
 
-    def _goto(self, follow_imports=False, follow_builtin_imports=False,
-              only_stubs=False, prefer_stubs=False):
+        """
+        Like :meth:`.Script.goto` (also supports the same params), but does it
+        for the current name. This is typically useful if you are using
+        something like :meth:`.Script.get_names()`.
 
+        :param follow_imports: The goto call will follow imports.
+        :param follow_builtin_imports: If follow_imports is True will try to
+            look up names in builtins (i.e. compiled or extension modules).
+        :param only_stubs: Only return stubs for this goto call.
+        :param prefer_stubs: Prefer stubs to Python objects for this goto call.
+        :rtype: list of :class:`Name`
+        """
         if not self._name.is_value_name:
             return []
 
@@ -383,14 +450,27 @@ class BaseDefinition(object):
             only_stubs=only_stubs,
             prefer_stubs=prefer_stubs,
         )
-        return [self if n == self._name else Definition(self._inference_state, n)
+        return [self if n == self._name else Name(self._inference_state, n)
                 for n in names]
 
-    def infer(self, **kwargs):  # Python 2...
-        with debug.increase_indent_cm('infer for %s' % self._name):
-            return self._infer(**kwargs)
+    @debug.increase_indent_cm('infer on name')
+    def infer(self, *, only_stubs=False, prefer_stubs=False):
+        """
+        Like :meth:`.Script.infer`, it can be useful to understand which type
+        the current name has.
 
-    def _infer(self, only_stubs=False, prefer_stubs=False):
+        Return the actual definitions. I strongly recommend not using it for
+        your completions, because it might slow down |jedi|. If you want to
+        read only a few objects (<=20), it might be useful, especially to get
+        the original docstrings. The basic problem of this function is that it
+        follows all results. This means with 1000 completions (e.g.  numpy),
+        it's just very, very slow.
+
+        :param only_stubs: Only return stubs for this goto call.
+        :param prefer_stubs: Prefer stubs to Python objects for this type
+            inference call.
+        :rtype: list of :class:`Name`
+        """
         assert not (only_stubs and prefer_stubs)
 
         if not self._name.is_value_name:
@@ -406,38 +486,15 @@ class BaseDefinition(object):
             prefer_stubs=prefer_stubs,
         )
         resulting_names = [c.name for c in values]
-        return [self if n == self._name else Definition(self._inference_state, n)
+        return [self if n == self._name else Name(self._inference_state, n)
                 for n in resulting_names]
 
-    @property
-    @memoize_method
-    def params(self):
-        """
-        Deprecated! Will raise a warning soon. Use get_signatures()[...].params.
-
-        Raises an ``AttributeError`` if the definition is not callable.
-        Otherwise returns a list of `Definition` that represents the params.
-        """
-        warnings.warn(
-            "Deprecated since version 0.16.0. Use get_signatures()[...].params",
-            DeprecationWarning,
-            stacklevel=2
-        )
-        # Only return the first one. There might be multiple one, especially
-        # with overloading.
-        for signature in self._get_signatures():
-            return [
-                Definition(self._inference_state, n)
-                for n in signature.get_param_names(resolve_stars=True)
-            ]
-
-        if self.type == 'function' or self.type == 'class':
-            # Fallback, if no signatures were defined (which is probably by
-            # itself a bug).
-            return []
-        raise AttributeError('There are no params defined on this.')
-
     def parent(self):
+        """
+        Returns the parent scope of this identifier.
+
+        :rtype: Name
+        """
         if not self._name.is_value_name:
             return None
 
@@ -463,7 +520,7 @@ class BaseDefinition(object):
             # Happens for comprehension contexts
             context = context.parent_context
 
-        return Definition(self._inference_state, context.name)
+        return Name(self._inference_state, context.name)
 
     def __repr__(self):
         return "<%s %sname=%r, description=%r>" % (
@@ -496,32 +553,64 @@ class BaseDefinition(object):
         return ''.join(lines[start_index:index + after + 1])
 
     def _get_signatures(self, for_docstring=False):
+        if self._name.api_type == 'property':
+            return []
         if for_docstring and self._name.api_type == 'statement' and not self.is_stub():
             # For docstrings we don't resolve signatures if they are simple
             # statements and not stubs. This is a speed optimization.
             return []
 
+        if isinstance(self._name, MixedName):
+            # While this would eventually happen anyway, it's basically just a
+            # shortcut to not infer anything tree related, because it's really
+            # not necessary.
+            return self._name.infer_compiled_value().get_signatures()
+
         names = convert_names([self._name], prefer_stubs=True)
         return [sig for name in names for sig in name.infer().get_signatures()]
 
     def get_signatures(self):
+        """
+        Returns all potential signatures for a function or a class. Multiple
+        signatures are typical if you use Python stubs with ``@overload``.
+
+        :rtype: list of :class:`BaseSignature`
+        """
         return [
             BaseSignature(self._inference_state, s)
             for s in self._get_signatures()
         ]
 
     def execute(self):
+        """
+        Uses type inference to "execute" this identifier and returns the
+        executed objects.
+
+        :rtype: list of :class:`Name`
+        """
         return _values_to_definitions(self._name.infer().execute_with_values())
 
+    def get_type_hint(self):
+        """
+        Returns type hints like ``Iterable[int]`` or ``Union[int, str]``.
 
-class Completion(BaseDefinition):
+        This method might be quite slow, especially for functions. The problem
+        is finding executions for those functions to return something like
+        ``Callable[[int, str], str]``.
+
+        :rtype: str
+        """
+        return self._name.infer().get_type_hint()
+
+
+class Completion(BaseName):
     """
-    `Completion` objects are returned from :meth:`api.Script.complete`. They
+    ``Completion`` objects are returned from :meth:`.Script.complete`. They
     provide additional information about a completion.
     """
     def __init__(self, inference_state, name, stack, like_name_length,
                  is_fuzzy, cached_name=None):
-        super(Completion, self).__init__(inference_state, name)
+        super().__init__(inference_state, name)
 
         self._like_name_length = like_name_length
         self._stack = stack
@@ -554,15 +643,15 @@ class Completion(BaseDefinition):
             isinstan# <-- Cursor is here
 
         would return the string 'ce'. It also adds additional stuff, depending
-        on your `settings.py`.
+        on your ``settings.py``.
 
         Assuming the following function definition::
 
             def foo(param=0):
                 pass
 
-        completing ``foo(par`` would give a ``Completion`` which `complete`
-        would be `am=`
+        completing ``foo(par`` would give a ``Completion`` which ``complete``
+        would be ``am=``.
         """
         if self._is_fuzzy:
             return None
@@ -571,7 +660,7 @@ class Completion(BaseDefinition):
     @property
     def name_with_symbols(self):
         """
-        Similar to :attr:`name`, but like :attr:`name` returns also the
+        Similar to :attr:`.name`, but like :attr:`.name` returns also the
         symbols, for example assuming the following function definition::
 
             def foo(param=0):
@@ -584,12 +673,15 @@ class Completion(BaseDefinition):
         return self._complete(False)
 
     def docstring(self, raw=False, fast=True):
+        """
+        Documented under :meth:`BaseName.docstring`.
+        """
         if self._like_name_length >= 3:
             # In this case we can just resolve the like name, because we
             # wouldn't load like > 100 Python modules anymore.
             fast = False
 
-        return super(Completion, self).docstring(raw=raw, fast=fast)
+        return super().docstring(raw=raw, fast=fast)
 
     def _get_docstring(self):
         if self._cached_name is not None:
@@ -598,7 +690,7 @@ class Completion(BaseDefinition):
                 self._name.get_public_name(),
                 lambda: self._get_cache()
             )
-        return super(Completion, self)._get_docstring()
+        return super()._get_docstring()
 
     def _get_docstring_signature(self):
         if self._cached_name is not None:
@@ -607,18 +699,20 @@ class Completion(BaseDefinition):
                 self._name.get_public_name(),
                 lambda: self._get_cache()
             )
-        return super(Completion, self)._get_docstring_signature()
+        return super()._get_docstring_signature()
 
     def _get_cache(self):
-        typ = super(Completion, self).type
         return (
-            typ,
-            super(Completion, self)._get_docstring_signature(),
-            super(Completion, self)._get_docstring(),
+            super().type,
+            super()._get_docstring_signature(),
+            super()._get_docstring(),
         )
 
     @property
     def type(self):
+        """
+        Documented under :meth:`BaseName.type`.
+        """
         # Purely a speed optimization.
         if self._cached_name is not None:
             return completion_cache.get_type(
@@ -627,63 +721,48 @@ class Completion(BaseDefinition):
                 lambda: self._get_cache()
             )
 
-        return super(Completion, self).type
+        return super().type
+
+    def get_completion_prefix_length(self):
+        """
+        Returns the length of the prefix being completed.
+        For example, completing ``isinstance``::
+
+            isinstan# <-- Cursor is here
+
+        would return 8, because len('isinstan') == 8.
+
+        Assuming the following function definition::
+
+            def foo(param=0):
+                pass
+
+        completing ``foo(par`` would return 3.
+        """
+        return self._like_name_length
 
     def __repr__(self):
         return '<%s: %s>' % (type(self).__name__, self._name.get_public_name())
 
-    @memoize_method
-    def follow_definition(self):
-        """
-        Deprecated!
 
-        Return the original definitions. I strongly recommend not using it for
-        your completions, because it might slow down |jedi|. If you want to
-        read only a few objects (<=20), it might be useful, especially to get
-        the original docstrings. The basic problem of this function is that it
-        follows all results. This means with 1000 completions (e.g.  numpy),
-        it's just PITA-slow.
-        """
-        warnings.warn(
-            "Deprecated since version 0.14.0. Use .infer.",
-            DeprecationWarning,
-            stacklevel=2
-        )
-        return self.infer()
-
-
-class Definition(BaseDefinition):
+class Name(BaseName):
     """
-    *Definition* objects are returned from :meth:`api.Script.goto`
-    or :meth:`api.Script.infer`.
+    *Name* objects are returned from many different APIs including
+    :meth:`.Script.goto` or :meth:`.Script.infer`.
     """
     def __init__(self, inference_state, definition):
-        super(Definition, self).__init__(inference_state, definition)
-
-    @property
-    def desc_with_module(self):
-        """
-        In addition to the definition, also return the module.
-
-        .. warning:: Don't use this function yet, its behaviour may change. If
-            you really need it, talk to me.
-
-        .. todo:: Add full path. This function is should return a
-            `module.class.function` path.
-        """
-        position = '' if self.in_builtin_module else '@%s' % self.line
-        return "%s:%s%s" % (self.module_name, self.description, position)
+        super().__init__(inference_state, definition)
 
     @memoize_method
     def defined_names(self):
         """
         List sub-definitions (e.g., methods in class).
 
-        :rtype: list of Definition
+        :rtype: list of :class:`Name`
         """
         defs = self._name.infer()
         return sorted(
-            unite(defined_names(self._inference_state, d.as_context()) for d in defs),
+            unite(defined_names(self._inference_state, d) for d in defs),
             key=lambda s: s._name.start_pos or (0, 0)
         )
 
@@ -710,45 +789,53 @@ class Definition(BaseDefinition):
         return hash((self._name.start_pos, self.module_path, self.name, self._inference_state))
 
 
-class BaseSignature(Definition):
+class BaseSignature(Name):
     """
-    `BaseSignature` objects is the return value of `Script.function_definition`.
-    It knows what functions you are currently in. e.g. `isinstance(` would
-    return the `isinstance` function. without `(` it would return nothing.
+    These signatures are returned by :meth:`BaseName.get_signatures`
+    calls.
     """
     def __init__(self, inference_state, signature):
-        super(BaseSignature, self).__init__(inference_state, signature.name)
+        super().__init__(inference_state, signature.name)
         self._signature = signature
 
     @property
     def params(self):
         """
-        :return list of ParamDefinition:
+        Returns definitions for all parameters that a signature defines.
+        This includes stuff like ``*args`` and ``**kwargs``.
+
+        :rtype: list of :class:`.ParamName`
         """
-        return [ParamDefinition(self._inference_state, n)
+        return [ParamName(self._inference_state, n)
                 for n in self._signature.get_param_names(resolve_stars=True)]
 
     def to_string(self):
+        """
+        Returns a text representation of the signature. This could for example
+        look like ``foo(bar, baz: int, **kwargs)``.
+
+        :rtype: str
+        """
         return self._signature.to_string()
 
 
 class Signature(BaseSignature):
     """
-    `Signature` objects is the return value of `Script.get_signatures`.
-    It knows what functions you are currently in. e.g. `isinstance(` would
-    return the `isinstance` function with its params. Without `(` it would
-    return nothing.
+    A full signature object is the return value of
+    :meth:`.Script.get_signatures`.
     """
     def __init__(self, inference_state, signature, call_details):
-        super(Signature, self).__init__(inference_state, signature)
+        super().__init__(inference_state, signature)
         self._call_details = call_details
         self._signature = signature
 
     @property
     def index(self):
         """
-        The Param index of the current call.
+        Returns the param index of the current cursor position.
         Returns None if the index cannot be found in the curent call.
+
+        :rtype: int
         """
         return self._call_details.calculate_index(
             self._signature.get_param_names(resolve_stars=True)
@@ -757,8 +844,10 @@ class Signature(BaseSignature):
     @property
     def bracket_start(self):
         """
-        The line/column of the bracket that is responsible for the last
-        function call.
+        Returns a line/column tuple of the bracket that is responsible for the
+        last function call. The first line is 1 and the first column 0.
+
+        :rtype: int, int
         """
         return self._call_details.bracket_leaf.start_pos
 
@@ -770,35 +859,37 @@ class Signature(BaseSignature):
         )
 
 
-class ParamDefinition(Definition):
+class ParamName(Name):
     def infer_default(self):
         """
-        :return list of Definition:
+        Returns default values like the ``1`` of ``def foo(x=1):``.
+
+        :rtype: list of :class:`.Name`
         """
         return _values_to_definitions(self._name.infer_default())
 
     def infer_annotation(self, **kwargs):
         """
-        :return list of Definition:
-
-        :param execute_annotation: If False, the values are not executed and
-            you get classes instead of instances.
+        :param execute_annotation: Default True; If False, values are not
+            executed and classes are returned instead of instances.
+        :rtype: list of :class:`.Name`
         """
         return _values_to_definitions(self._name.infer_annotation(ignore_stars=True, **kwargs))
 
     def to_string(self):
+        """
+        Returns a simple representation of a param, like
+        ``f: Callable[..., Any]``.
+
+        :rtype: str
+        """
         return self._name.to_string()
 
     @property
     def kind(self):
         """
-        Returns an enum instance. Returns the same values as the builtin
-        :py:attr:`inspect.Parameter.kind`.
+        Returns an enum instance of :mod:`inspect`'s ``Parameter`` enum.
 
-        No support for Python < 3.4 anymore.
+        :rtype: :py:attr:`inspect.Parameter.kind`
         """
-        if sys.version_info < (3, 5):
-            raise NotImplementedError(
-                'Python 2 is end-of-life, the new feature is not available for it'
-            )
         return self._name.get_kind()

jedi/api/completion.py

@@ -1,12 +1,12 @@
 import re
 from textwrap import dedent
+from inspect import Parameter
 
 from parso.python.token import PythonTokenTypes
 from parso.python import tree
 from parso.tree import search_ancestor, Leaf
 from parso import split_lines
 
-from jedi._compatibility import Parameter
 from jedi import debug
 from jedi import settings
 from jedi.api import classes
@@ -18,9 +18,10 @@ from jedi.inference import imports
 from jedi.inference.base_value import ValueSet
 from jedi.inference.helpers import infer_call_of_leaf, parse_dotted_names
 from jedi.inference.context import get_global_filters
-from jedi.inference.value import TreeInstance, ModuleValue
-from jedi.inference.names import ParamNameWrapper
-from jedi.inference.gradual.conversion import convert_values
+from jedi.inference.value import TreeInstance
+from jedi.inference.docstring_utils import DocstringModule
+from jedi.inference.names import ParamNameWrapper, SubModuleName
+from jedi.inference.gradual.conversion import convert_values, convert_names
 from jedi.parser_utils import cut_value_at_position
 from jedi.plugins import plugin_manager
 
@@ -30,29 +31,52 @@ class ParamNameWithEquals(ParamNameWrapper):
         return self.string_name + '='
 
 
-def get_signature_param_names(signatures):
-    # add named params
+def _get_signature_param_names(signatures, positional_count, used_kwargs):
+    # Add named params
     for call_sig in signatures:
-        for p in call_sig.params:
-            # Allow protected access, because it's a public API.
-            if p._name.get_kind() in (Parameter.POSITIONAL_OR_KEYWORD,
-                                      Parameter.KEYWORD_ONLY):
+        for i, p in enumerate(call_sig.params):
+            kind = p.kind
+            if i < positional_count and kind == Parameter.POSITIONAL_OR_KEYWORD:
+                continue
+            if kind in (Parameter.POSITIONAL_OR_KEYWORD, Parameter.KEYWORD_ONLY) \
+                    and p.name not in used_kwargs:
                 yield ParamNameWithEquals(p._name)
 
 
-def filter_names(inference_state, completion_names, stack, like_name, fuzzy, cached_name):
+def _must_be_kwarg(signatures, positional_count, used_kwargs):
+    if used_kwargs:
+        return True
+
+    must_be_kwarg = True
+    for signature in signatures:
+        for i, p in enumerate(signature.params):
+            kind = p.kind
+            if kind is Parameter.VAR_POSITIONAL:
+                # In case there were not already kwargs, the next param can
+                # always be a normal argument.
+                return False
+
+            if i >= positional_count and kind in (Parameter.POSITIONAL_OR_KEYWORD,
+                                                  Parameter.POSITIONAL_ONLY):
+                must_be_kwarg = False
+                break
+        if not must_be_kwarg:
+            break
+    return must_be_kwarg
+
+
+def filter_names(inference_state, completion_names, stack, like_name, fuzzy,
+                 imported_names, cached_name):
     comp_dct = set()
     if settings.case_insensitive_completion:
         like_name = like_name.lower()
     for name in completion_names:
         string = name.string_name
+        if string in imported_names and string != like_name:
+            continue
         if settings.case_insensitive_completion:
             string = string.lower()
-        if fuzzy:
-            match = helpers.fuzzy_match(string, like_name)
-        else:
-            match = helpers.start_match(string, like_name)
-        if match:
+        if helpers.match(string, like_name, fuzzy=fuzzy):
             new = classes.Completion(
                 inference_state,
                 name,
@@ -117,6 +141,11 @@ class Completion:
 
         self._fuzzy = fuzzy
 
+    # Return list of completions in this order:
+    # - Beginning with what user is typing
+    # - Public (alphabet)
+    # - Private ("_xxx")
+    # - Dunder ("__xxx")
     def complete(self):
         leaf = self._module_node.get_leaf_for_position(
             self._original_position,
@@ -135,7 +164,7 @@ class Completion:
 
         if string is not None and not prefixed_completions:
             prefixed_completions = list(complete_file_name(
-                self._inference_state, self._module_context, start_leaf, string,
+                self._inference_state, self._module_context, start_leaf, quote, string,
                 self._like_name, self._signatures_callback,
                 self._code_lines, self._original_position,
                 self._fuzzy
@@ -148,14 +177,19 @@ class Completion:
 
         cached_name, completion_names = self._complete_python(leaf)
 
+        imported_names = []
+        if leaf.parent is not None and leaf.parent.type in ['import_as_names', 'dotted_as_names']:
+            imported_names.extend(extract_imported_names(leaf.parent))
+
         completions = list(filter_names(self._inference_state, completion_names,
                                         self.stack, self._like_name,
-                                        self._fuzzy, cached_name=cached_name))
+                                        self._fuzzy, imported_names, cached_name=cached_name))
 
         return (
             # Removing duplicates mostly to remove False/True/None duplicates.
             _remove_duplicates(prefixed_completions, completions)
-            + sorted(completions, key=lambda x: (x.name.startswith('__'),
+            + sorted(completions, key=lambda x: (not x.name.startswith(self._like_name),
+                                                 x.name.startswith('__'),
                                                  x.name.startswith('_'),
                                                  x.name.lower()))
         )
@@ -174,7 +208,6 @@ class Completion:
         - In args: */**: no completion
         - In params (also lambda): no completion before =
         """
-
         grammar = self._inference_state.grammar
         self.stack = stack = None
         self._position = (
@@ -233,14 +266,8 @@ class Completion:
                             allowed_transitions.append('else')
 
         completion_names = []
-        current_line = self._code_lines[self._position[0] - 1][:self._position[1]]
-
-        completion_names += self._complete_keywords(
-            allowed_transitions,
-            only_values=not (not current_line or current_line[-1] in ' \t.;'
-                             and current_line[-3:] != '...')
-        )
 
+        kwargs_only = False
         if any(t in allowed_transitions for t in (PythonTokenTypes.NAME,
                                                   PythonTokenTypes.INDENT)):
             # This means that we actually have to do type inference.
@@ -263,25 +290,50 @@ class Completion:
                 )
             elif nonterminals[-1] in ('trailer', 'dotted_name') and nodes[-1] == '.':
                 dot = self._module_node.get_leaf_for_position(self._position)
+                if dot.type == "endmarker":
+                    # This is a bit of a weird edge case, maybe we can somehow
+                    # generalize this.
+                    dot = leaf.get_previous_leaf()
                 cached_name, n = self._complete_trailer(dot.get_previous_leaf())
                 completion_names += n
             elif self._is_parameter_completion():
                 completion_names += self._complete_params(leaf)
             else:
-                completion_names += self._complete_global_scope()
-                completion_names += self._complete_inherited(is_function=False)
+                # Apparently this looks like it's good enough to filter most cases
+                # so that signature completions don't randomly appear.
+                # To understand why this works, three things are important:
+                # 1. trailer with a `,` in it is either a subscript or an arglist.
+                # 2. If there's no `,`, it's at the start and only signatures start
+                #    with `(`. Other trailers could start with `.` or `[`.
+                # 3. Decorators are very primitive and have an optional `(` with
+                #    optional arglist in them.
+                if nodes[-1] in ['(', ','] \
+                        and nonterminals[-1] in ('trailer', 'arglist', 'decorator'):
+                    signatures = self._signatures_callback(*self._position)
+                    if signatures:
+                        call_details = signatures[0]._call_details
+                        used_kwargs = list(call_details.iter_used_keyword_arguments())
+                        positional_count = call_details.count_positional_arguments()
 
-            # Apparently this looks like it's good enough to filter most cases
-            # so that signature completions don't randomly appear.
-            # To understand why this works, three things are important:
-            # 1. trailer with a `,` in it is either a subscript or an arglist.
-            # 2. If there's no `,`, it's at the start and only signatures start
-            #    with `(`. Other trailers could start with `.` or `[`.
-            # 3. Decorators are very primitive and have an optional `(` with
-            #    optional arglist in them.
-            if nodes[-1] in ['(', ','] and nonterminals[-1] in ('trailer', 'arglist', 'decorator'):
-                signatures = self._signatures_callback(*self._position)
-                completion_names += get_signature_param_names(signatures)
+                        completion_names += _get_signature_param_names(
+                            signatures,
+                            positional_count,
+                            used_kwargs,
+                        )
+
+                        kwargs_only = _must_be_kwarg(signatures, positional_count, used_kwargs)
+
+                if not kwargs_only:
+                    completion_names += self._complete_global_scope()
+                    completion_names += self._complete_inherited(is_function=False)
+
+        if not kwargs_only:
+            current_line = self._code_lines[self._position[0] - 1][:self._position[1]]
+            completion_names += self._complete_keywords(
+                allowed_transitions,
+                only_values=not (not current_line or current_line[-1] in ' \t.;'
+                                 and current_line[-3:] != '...')
+            )
 
         return cached_name, completion_names
 
@@ -305,19 +357,20 @@ class Completion:
         if stack_node.nonterminal == 'funcdef':
             context = get_user_context(self._module_context, self._position)
             node = search_ancestor(leaf, 'error_node', 'funcdef')
-            if node.type == 'error_node':
-                n = node.children[0]
-                if n.type == 'decorators':
-                    decorators = n.children
-                elif n.type == 'decorator':
-                    decorators = [n]
+            if node is not None:
+                if node.type == 'error_node':
+                    n = node.children[0]
+                    if n.type == 'decorators':
+                        decorators = n.children
+                    elif n.type == 'decorator':
+                        decorators = [n]
+                    else:
+                        decorators = []
                 else:
-                    decorators = []
-            else:
-                decorators = node.get_decorators()
-            function_name = stack_node.nodes[1]
+                    decorators = node.get_decorators()
+                function_name = stack_node.nodes[1]
 
-            return complete_param_names(context, function_name.value, decorators)
+                return complete_param_names(context, function_name.value, decorators)
         return []
 
     def _complete_keywords(self, allowed_transitions, only_values):
@@ -361,80 +414,7 @@ class Completion:
     def _complete_trailer_for_values(self, values):
         user_context = get_user_context(self._module_context, self._position)
 
-        completion_names = []
-        for value in values:
-            for filter in value.get_filters(origin_scope=user_context.tree_node):
-                completion_names += filter.values()
-
-            if not value.is_stub() and isinstance(value, TreeInstance):
-                completion_names += self._complete_getattr(value)
-
-        python_values = convert_values(values)
-        for c in python_values:
-            if c not in values:
-                for filter in c.get_filters(origin_scope=user_context.tree_node):
-                    completion_names += filter.values()
-        return completion_names
-
-    def _complete_getattr(self, instance):
-        """
-        A heuristic to make completion for proxy objects work. This is not
-        intended to work in all cases. It works exactly in this case:
-
-            def __getattr__(self, name):
-                ...
-                return getattr(any_object, name)
-
-        It is important that the return contains getattr directly, otherwise it
-        won't work anymore. It's really just a stupid heuristic. It will not
-        work if you write e.g. `return (getatr(o, name))`, because of the
-        additional parentheses. It will also not work if you move the getattr
-        to some other place that is not the return statement itself.
-
-        It is intentional that it doesn't work in all cases. Generally it's
-        really hard to do even this case (as you can see below). Most people
-        will write it like this anyway and the other ones, well they are just
-        out of luck I guess :) ~dave.
-        """
-        names = (instance.get_function_slot_names(u'__getattr__')
-                 or instance.get_function_slot_names(u'__getattribute__'))
-        functions = ValueSet.from_sets(
-            name.infer()
-            for name in names
-        )
-        for func in functions:
-            tree_node = func.tree_node
-            for return_stmt in tree_node.iter_return_stmts():
-                # Basically until the next comment we just try to find out if a
-                # return statement looks exactly like `return getattr(x, name)`.
-                if return_stmt.type != 'return_stmt':
-                    continue
-                atom_expr = return_stmt.children[1]
-                if atom_expr.type != 'atom_expr':
-                    continue
-                atom = atom_expr.children[0]
-                trailer = atom_expr.children[1]
-                if len(atom_expr.children) != 2 or atom.type != 'name' \
-                        or atom.value != 'getattr':
-                    continue
-                arglist = trailer.children[1]
-                if arglist.type != 'arglist' or len(arglist.children) < 3:
-                    continue
-                context = func.as_context()
-                object_node = arglist.children[0]
-
-                # Make sure it's a param: foo in __getattr__(self, foo)
-                name_node = arglist.children[2]
-                name_list = context.goto(name_node, name_node.start_pos)
-                if not any(n.api_type == 'param' for n in name_list):
-                    continue
-
-                # Now that we know that these are most probably completion
-                # objects, we just infer the object and return them as
-                # completions.
-                objects = context.infer_node(object_node)
-                return self._complete_trailer_for_values(objects)
-        return []
+        return complete_trailer(user_context, values)
 
     def _get_importer_names(self, names, level=0, only_modules=True):
         names = [n.value for n in names]
@@ -490,7 +470,7 @@ class Completion:
         relevant_code_lines = list(iter_relevant_lines(code_lines))
         if relevant_code_lines[-1] is not None:
             # Some code lines might be None, therefore get rid of that.
-            relevant_code_lines = [c or '\n' for c in relevant_code_lines]
+            relevant_code_lines = ['\n' if c is None else c for c in relevant_code_lines]
             return self._complete_code_lines(relevant_code_lines)
         match = re.search(r'`([^`\s]+)', code_lines[-1])
         if match:
@@ -499,12 +479,12 @@ class Completion:
 
     def _complete_code_lines(self, code_lines):
         module_node = self._inference_state.grammar.parse(''.join(code_lines))
-        module_value = ModuleValue(
-            self._inference_state,
-            module_node,
+        module_value = DocstringModule(
+            in_module_context=self._module_context,
+            inference_state=self._inference_state,
+            module_node=module_node,
             code_lines=code_lines,
         )
-        module_value.parent_context = self._module_context
         return Completion(
             self._inference_state,
             module_value.as_context(),
@@ -534,6 +514,8 @@ def _extract_string_while_in_string(leaf, position):
         if leaf.line == position[0]:
             kwargs['endpos'] = position[1] - leaf.column
         match = _string_start.match(leaf.value, **kwargs)
+        if not match:
+            return None, None, None
         start = match.group(0)
         if leaf.line == position[0] and position[1] < leaf.column + match.end():
             return None, None, None
@@ -572,3 +554,139 @@ def _extract_string_while_in_string(leaf, position):
         leaves.insert(0, leaf)
         leaf = leaf.get_previous_leaf()
     return None, None, None
+
+
+def complete_trailer(user_context, values):
+    completion_names = []
+    for value in values:
+        for filter in value.get_filters(origin_scope=user_context.tree_node):
+            completion_names += filter.values()
+
+        if not value.is_stub() and isinstance(value, TreeInstance):
+            completion_names += _complete_getattr(user_context, value)
+
+    python_values = convert_values(values)
+    for c in python_values:
+        if c not in values:
+            for filter in c.get_filters(origin_scope=user_context.tree_node):
+                completion_names += filter.values()
+    return completion_names
+
+
+def _complete_getattr(user_context, instance):
+    """
+    A heuristic to make completion for proxy objects work. This is not
+    intended to work in all cases. It works exactly in this case:
+
+        def __getattr__(self, name):
+            ...
+            return getattr(any_object, name)
+
+    It is important that the return contains getattr directly, otherwise it
+    won't work anymore. It's really just a stupid heuristic. It will not
+    work if you write e.g. `return (getatr(o, name))`, because of the
+    additional parentheses. It will also not work if you move the getattr
+    to some other place that is not the return statement itself.
+
+    It is intentional that it doesn't work in all cases. Generally it's
+    really hard to do even this case (as you can see below). Most people
+    will write it like this anyway and the other ones, well they are just
+    out of luck I guess :) ~dave.
+    """
+    names = (instance.get_function_slot_names('__getattr__')
+             or instance.get_function_slot_names('__getattribute__'))
+    functions = ValueSet.from_sets(
+        name.infer()
+        for name in names
+    )
+    for func in functions:
+        tree_node = func.tree_node
+        if tree_node is None or tree_node.type != 'funcdef':
+            continue
+
+        for return_stmt in tree_node.iter_return_stmts():
+            # Basically until the next comment we just try to find out if a
+            # return statement looks exactly like `return getattr(x, name)`.
+            if return_stmt.type != 'return_stmt':
+                continue
+            atom_expr = return_stmt.children[1]
+            if atom_expr.type != 'atom_expr':
+                continue
+            atom = atom_expr.children[0]
+            trailer = atom_expr.children[1]
+            if len(atom_expr.children) != 2 or atom.type != 'name' \
+                    or atom.value != 'getattr':
+                continue
+            arglist = trailer.children[1]
+            if arglist.type != 'arglist' or len(arglist.children) < 3:
+                continue
+            context = func.as_context()
+            object_node = arglist.children[0]
+
+            # Make sure it's a param: foo in __getattr__(self, foo)
+            name_node = arglist.children[2]
+            name_list = context.goto(name_node, name_node.start_pos)
+            if not any(n.api_type == 'param' for n in name_list):
+                continue
+
+            # Now that we know that these are most probably completion
+            # objects, we just infer the object and return them as
+            # completions.
+            objects = context.infer_node(object_node)
+            return complete_trailer(user_context, objects)
+    return []
+
+
+def search_in_module(inference_state, module_context, names, wanted_names,
+                     wanted_type, complete=False, fuzzy=False,
+                     ignore_imports=False, convert=False):
+    for s in wanted_names[:-1]:
+        new_names = []
+        for n in names:
+            if s == n.string_name:
+                if n.tree_name is not None and n.api_type in ('module', 'namespace') \
+                        and ignore_imports:
+                    continue
+                new_names += complete_trailer(
+                    module_context,
+                    n.infer()
+                )
+        debug.dbg('dot lookup on search %s from %s', new_names, names[:10])
+        names = new_names
+
+    last_name = wanted_names[-1].lower()
+    for n in names:
+        string = n.string_name.lower()
+        if complete and helpers.match(string, last_name, fuzzy=fuzzy) \
+                or not complete and string == last_name:
+            if isinstance(n, SubModuleName):
+                names = [v.name for v in n.infer()]
+            else:
+                names = [n]
+            if convert:
+                names = convert_names(names)
+            for n2 in names:
+                if complete:
+                    def_ = classes.Completion(
+                        inference_state, n2,
+                        stack=None,
+                        like_name_length=len(last_name),
+                        is_fuzzy=fuzzy,
+                    )
+                else:
+                    def_ = classes.Name(inference_state, n2)
+                if not wanted_type or wanted_type == def_.type:
+                    yield def_
+
+
+def extract_imported_names(node):
+    imported_names = []
+
+    if node.type in ['import_as_names', 'dotted_as_names', 'import_as_name']:
+        for child in node.children:
+            if child.type == 'name':
+                imported_names.append(child.value)
+            elif child.type == 'import_as_name':
+                imported_names.extend(extract_imported_names(child))
+
+    return imported_names

jedi/api/completion_cache.py

@@ -1,7 +1,13 @@
-_cache = {}
+from typing import Dict, Tuple, Callable
+
+CacheValues = Tuple[str, str, str]
+CacheValuesCallback = Callable[[], CacheValues]
 
 
-def save_entry(module_name, name, cache):
+_cache: Dict[str, Dict[str, CacheValues]] = {}
+
+
+def save_entry(module_name: str, name: str, cache: CacheValues) -> None:
     try:
         module_cache = _cache[module_name]
     except KeyError:
@@ -9,8 +15,8 @@ def save_entry(module_name, name, cache):
     module_cache[name] = cache
 
 
-def _create_get_from_cache(number):
-    def _get_from_cache(module_name, name, get_cache_values):
+def _create_get_from_cache(number: int) -> Callable[[str, str, CacheValuesCallback], str]:
+    def _get_from_cache(module_name: str, name: str, get_cache_values: CacheValuesCallback) -> str:
         try:
             return _cache[module_name][name][number]
         except KeyError:

jedi/api/environment.py

@@ -7,17 +7,22 @@ import sys
 import hashlib
 import filecmp
 from collections import namedtuple
+from shutil import which
+from typing import TYPE_CHECKING
 
-from jedi._compatibility import highest_pickle_protocol, which
 from jedi.cache import memoize_method, time_cache
 from jedi.inference.compiled.subprocess import CompiledSubprocess, \
     InferenceStateSameProcess, InferenceStateSubprocess
 
 import parso
 
-_VersionInfo = namedtuple('VersionInfo', 'major minor micro')
+if TYPE_CHECKING:
+    from jedi.inference import InferenceState
 
-_SUPPORTED_PYTHONS = ['3.8', '3.7', '3.6', '3.5', '3.4', '2.7']
+
+_VersionInfo = namedtuple('VersionInfo', 'major minor micro')  # type: ignore[name-match]
+
+_SUPPORTED_PYTHONS = ['3.13', '3.12', '3.11', '3.10', '3.9', '3.8', '3.7', '3.6']
 _SAFE_PATHS = ['/usr/bin', '/usr/local/bin']
 _CONDA_VAR = 'CONDA_PREFIX'
 _CURRENT_VERSION = '%s.%s' % (sys.version_info.major, sys.version_info.minor)
@@ -30,7 +35,7 @@ class InvalidPythonEnvironment(Exception):
     """
 
 
-class _BaseEnvironment(object):
+class _BaseEnvironment:
     @memoize_method
     def get_grammar(self):
         version_string = '%s.%s' % (self.version_info.major, self.version_info.minor)
@@ -61,8 +66,9 @@ class Environment(_BaseEnvironment):
     """
     _subprocess = None
 
-    def __init__(self, executable):
+    def __init__(self, executable, env_vars=None):
         self._start_executable = executable
+        self._env_vars = env_vars
         # Initialize the environment
         self._get_subprocess()
 
@@ -71,7 +77,8 @@ class Environment(_BaseEnvironment):
             return self._subprocess
 
         try:
-            self._subprocess = CompiledSubprocess(self._start_executable)
+            self._subprocess = CompiledSubprocess(self._start_executable,
+                                                  env_vars=self._env_vars)
             info = self._subprocess._send(None, _get_info)
         except Exception as exc:
             raise InvalidPythonEnvironment(
@@ -91,33 +98,26 @@ class Environment(_BaseEnvironment):
         """
         self.version_info = _VersionInfo(*info[2])
         """
-        Like ``sys.version_info``. A tuple to show the current Environment's
-        Python version.
+        Like :data:`sys.version_info`: a tuple to show the current
+        Environment's Python version.
         """
-
-        # py2 sends bytes via pickle apparently?!
-        if self.version_info.major == 2:
-            self.executable = self.executable.decode()
-            self.path = self.path.decode()
-
-        # Adjust pickle protocol according to host and client version.
-        self._subprocess._pickle_protocol = highest_pickle_protocol([
-            sys.version_info, self.version_info])
-
         return self._subprocess
 
     def __repr__(self):
         version = '.'.join(str(i) for i in self.version_info)
         return '<%s: %s in %s>' % (self.__class__.__name__, version, self.path)
 
-    def get_inference_state_subprocess(self, inference_state):
+    def get_inference_state_subprocess(
+        self,
+        inference_state: 'InferenceState',
+    ) -> InferenceStateSubprocess:
         return InferenceStateSubprocess(inference_state, self._get_subprocess())
 
     @memoize_method
     def get_sys_path(self):
         """
         The sys path for this environment. Does not include potential
-        modifications like ``sys.path.append``.
+        modifications from e.g. appending to :data:`sys.path`.
 
         :returns: list of str
         """
@@ -129,11 +129,12 @@ class Environment(_BaseEnvironment):
         return self._get_subprocess().get_sys_path()
 
 
-class _SameEnvironmentMixin(object):
+class _SameEnvironmentMixin:
     def __init__(self):
         self._start_executable = self.executable = sys.executable
         self.path = sys.prefix
         self.version_info = _VersionInfo(*sys.version_info[:3])
+        self._env_vars = None
 
 
 class SameEnvironment(_SameEnvironmentMixin, Environment):
@@ -141,7 +142,10 @@ class SameEnvironment(_SameEnvironmentMixin, Environment):
 
 
 class InterpreterEnvironment(_SameEnvironmentMixin, _BaseEnvironment):
-    def get_inference_state_subprocess(self, inference_state):
+    def get_inference_state_subprocess(
+        self,
+        inference_state: 'InferenceState',
+    ) -> InferenceStateSameProcess:
         return InferenceStateSameProcess(inference_state)
 
     def get_sys_path(self):
@@ -185,7 +189,7 @@ def get_default_environment():
     makes it possible to use as many new Python features as possible when using
     autocompletion and other functionality.
 
-    :returns: :class:`Environment`
+    :returns: :class:`.Environment`
     """
     virtual_env = _get_virtual_env_from_var()
     if virtual_env is not None:
@@ -254,10 +258,17 @@ def get_cached_default_environment():
 
 @time_cache(seconds=10 * 60)  # 10 Minutes
 def _get_cached_default_environment():
-    return get_default_environment()
+    try:
+        return get_default_environment()
+    except InvalidPythonEnvironment:
+        # It's possible that `sys.executable` is wrong. Typically happens
+        # when Jedi is used in an executable that embeds Python. For further
+        # information, have a look at:
+        # https://github.com/davidhalter/jedi/issues/1531
+        return InterpreterEnvironment()
 
 
-def find_virtualenvs(paths=None, **kwargs):
+def find_virtualenvs(paths=None, *, safe=True, use_environment_vars=True):
     """
     :param paths: A list of paths in your file system to be scanned for
         Virtualenvs. It will search in these paths and potentially execute the
@@ -272,49 +283,46 @@ def find_virtualenvs(paths=None, **kwargs):
         CONDA_PREFIX will be checked to see if it contains a valid conda
         environment.
 
-    :yields: :class:`Environment`
+    :yields: :class:`.Environment`
     """
-    def py27_comp(paths=None, safe=True, use_environment_vars=True):
-        if paths is None:
-            paths = []
+    if paths is None:
+        paths = []
 
-        _used_paths = set()
+    _used_paths = set()
 
-        if use_environment_vars:
-            # Using this variable should be safe, because attackers might be
-            # able to drop files (via git) but not environment variables.
-            virtual_env = _get_virtual_env_from_var()
-            if virtual_env is not None:
-                yield virtual_env
-                _used_paths.add(virtual_env.path)
+    if use_environment_vars:
+        # Using this variable should be safe, because attackers might be
+        # able to drop files (via git) but not environment variables.
+        virtual_env = _get_virtual_env_from_var()
+        if virtual_env is not None:
+            yield virtual_env
+            _used_paths.add(virtual_env.path)
 
-            conda_env = _get_virtual_env_from_var(_CONDA_VAR)
-            if conda_env is not None:
-                yield conda_env
-                _used_paths.add(conda_env.path)
+        conda_env = _get_virtual_env_from_var(_CONDA_VAR)
+        if conda_env is not None:
+            yield conda_env
+            _used_paths.add(conda_env.path)
 
-        for directory in paths:
-            if not os.path.isdir(directory):
+    for directory in paths:
+        if not os.path.isdir(directory):
+            continue
+
+        directory = os.path.abspath(directory)
+        for path in os.listdir(directory):
+            path = os.path.join(directory, path)
+            if path in _used_paths:
+                # A path shouldn't be inferred twice.
                 continue
+            _used_paths.add(path)
 
-            directory = os.path.abspath(directory)
-            for path in os.listdir(directory):
-                path = os.path.join(directory, path)
-                if path in _used_paths:
-                    # A path shouldn't be inferred twice.
-                    continue
-                _used_paths.add(path)
-
-                try:
-                    executable = _get_executable_path(path, safe=safe)
-                    yield Environment(executable)
-                except InvalidPythonEnvironment:
-                    pass
-
-    return py27_comp(paths, **kwargs)
+            try:
+                executable = _get_executable_path(path, safe=safe)
+                yield Environment(executable)
+            except InvalidPythonEnvironment:
+                pass
 
 
-def find_system_environments():
+def find_system_environments(*, env_vars=None):
     """
     Ignores virtualenvs and returns the Python versions that were installed on
     your system. This might return nothing, if you're running Python e.g. from
@@ -322,24 +330,24 @@ def find_system_environments():
 
     The environments are sorted from latest to oldest Python version.
 
-    :yields: :class:`Environment`
+    :yields: :class:`.Environment`
     """
     for version_string in _SUPPORTED_PYTHONS:
         try:
-            yield get_system_environment(version_string)
+            yield get_system_environment(version_string, env_vars=env_vars)
         except InvalidPythonEnvironment:
             pass
 
 
 # TODO: this function should probably return a list of environments since
 # multiple Python installations can be found on a system for the same version.
-def get_system_environment(version):
+def get_system_environment(version, *, env_vars=None):
     """
     Return the first Python environment found for a string of the form 'X.Y'
     where X and Y are the major and minor versions of Python.
 
     :raises: :exc:`.InvalidPythonEnvironment`
-    :returns: :class:`Environment`
+    :returns: :class:`.Environment`
     """
     exe = which('python' + version)
     if exe:
@@ -350,24 +358,24 @@ def get_system_environment(version):
     if os.name == 'nt':
         for exe in _get_executables_from_windows_registry(version):
             try:
-                return Environment(exe)
+                return Environment(exe, env_vars=env_vars)
             except InvalidPythonEnvironment:
                 pass
     raise InvalidPythonEnvironment("Cannot find executable python%s." % version)
 
 
-def create_environment(path, safe=True):
+def create_environment(path, *, safe=True, env_vars=None):
     """
     Make it possible to manually create an Environment object by specifying a
-    Virtualenv path or an executable path.
+    Virtualenv path or an executable path and optional environment variables.
 
     :raises: :exc:`.InvalidPythonEnvironment`
-    :returns: :class:`Environment`
+    :returns: :class:`.Environment`
     """
     if os.path.isfile(path):
         _assert_safe(path, safe)
-        return Environment(path)
-    return Environment(_get_executable_path(path, safe=safe))
+        return Environment(path, env_vars=env_vars)
+    return Environment(_get_executable_path(path, safe=safe), env_vars=env_vars)
 
 
 def _get_executable_path(path, safe=True):
@@ -376,10 +384,13 @@ def _get_executable_path(path, safe=True):
     """
 
     if os.name == 'nt':
-        python = os.path.join(path, 'Scripts', 'python.exe')
+        pythons = [os.path.join(path, 'Scripts', 'python.exe'), os.path.join(path, 'python.exe')]
+    else:
+        pythons = [os.path.join(path, 'bin', 'python')]
+    for python in pythons:
+        if os.path.exists(python):
+            break
     else:
-        python = os.path.join(path, 'bin', 'python')
-    if not os.path.exists(python):
         raise InvalidPythonEnvironment("%s seems to be missing." % python)
 
     _assert_safe(python, safe)
@@ -387,11 +398,7 @@ def _get_executable_path(path, safe=True):
 
 
 def _get_executables_from_windows_registry(version):
-    # The winreg module is named _winreg on Python 2.
-    try:
-        import winreg
-    except ImportError:
-        import _winreg as winreg
+    import winreg
 
     # TODO: support Python Anaconda.
     sub_keys = [

jedi/api/errors.py

@@ -0,0 +1,46 @@
+"""
+This file is about errors in Python files and not about exception handling in
+Jedi.
+"""
+
+
+def parso_to_jedi_errors(grammar, module_node):
+    return [SyntaxError(e) for e in grammar.iter_errors(module_node)]
+
+
+class SyntaxError:
+    """
+    Syntax errors are generated by :meth:`.Script.get_syntax_errors`.
+    """
+    def __init__(self, parso_error):
+        self._parso_error = parso_error
+
+    @property
+    def line(self):
+        """The line where the error starts (starting with 1)."""
+        return self._parso_error.start_pos[0]
+
+    @property
+    def column(self):
+        """The column where the error starts (starting with 0)."""
+        return self._parso_error.start_pos[1]
+
+    @property
+    def until_line(self):
+        """The line where the error ends (starting with 1)."""
+        return self._parso_error.end_pos[0]
+
+    @property
+    def until_column(self):
+        """The column where the error ends (starting with 0)."""
+        return self._parso_error.end_pos[1]
+
+    def get_message(self):
+        return self._parso_error.message
+
+    def __repr__(self):
+        return '<%s from=%s to=%s>' % (
+            self.__class__.__name__,
+            self._parso_error.start_pos,
+            self._parso_error.end_pos,
+        )

jedi/api/exceptions.py

@@ -3,8 +3,29 @@ class _JediError(Exception):
 
 
 class InternalError(_JediError):
-    pass
+    """
+    This error might happen a subprocess is crashing. The reason for this is
+    usually broken C code in third party libraries. This is not a very common
+    thing and it is safe to use Jedi again. However using the same calls might
+    result in the same error again.
+    """
 
 
 class WrongVersion(_JediError):
-    pass
+    """
+    This error is reserved for the future, shouldn't really be happening at the
+    moment.
+    """
+
+
+class RefactoringError(_JediError):
+    """
+    Refactorings can fail for various reasons. So if you work with refactorings
+    like :meth:`.Script.rename`, :meth:`.Script.inline`,
+    :meth:`.Script.extract_variable` and :meth:`.Script.extract_function`, make
+    sure to catch these. The descriptions in the errors are usually valuable
+    for end users.
+
+    A typical ``RefactoringError`` would tell the user that inlining is not
+    possible if no name is under the cursor.
+    """

jedi/api/file_name.py

@@ -1,22 +1,23 @@
 import os
 
-from jedi._compatibility import FileNotFoundError, force_unicode, scandir
 from jedi.api import classes
 from jedi.api.strings import StringName, get_quote_ending
-from jedi.api.helpers import fuzzy_match, start_match
+from jedi.api.helpers import match
 from jedi.inference.helpers import get_str_or_none
 
 
 class PathName(StringName):
-    api_type = u'path'
+    api_type = 'path'
 
 
-def complete_file_name(inference_state, module_context, start_leaf, string,
+def complete_file_name(inference_state, module_context, start_leaf, quote, string,
                        like_name, signatures_callback, code_lines, position, fuzzy):
     # First we want to find out what can actually be changed as a name.
     like_name_length = len(os.path.basename(string))
 
     addition = _get_string_additions(module_context, start_leaf)
+    if string.startswith('~'):
+        string = os.path.expanduser(string)
     if addition is None:
         return
     string = addition + string
@@ -34,21 +35,18 @@ def complete_file_name(inference_state, module_context, start_leaf, string,
             is_in_os_path_join = False
         else:
             string = to_be_added + string
-    base_path = os.path.join(inference_state.project._path, string)
+    base_path = os.path.join(inference_state.project.path, string)
     try:
-        listed = sorted(scandir(base_path), key=lambda e: e.name)
+        listed = sorted(os.scandir(base_path), key=lambda e: e.name)
         # OSError: [Errno 36] File name too long: '...'
     except (FileNotFoundError, OSError):
         return
+    quote_ending = get_quote_ending(quote, code_lines, position)
     for entry in listed:
         name = entry.name
-        if fuzzy:
-            match = fuzzy_match(name, must_start_with)
-        else:
-            match = start_match(name, must_start_with)
-        if match:
+        if match(name, must_start_with, fuzzy=fuzzy):
             if is_in_os_path_join or not entry.is_dir():
-                name += get_quote_ending(start_leaf.value, code_lines, position)
+                name += quote_ending
             else:
                 name += os.path.sep
 
@@ -95,7 +93,7 @@ def _add_strings(context, nodes, add_slash=False):
             return None
         if not first and add_slash:
             string += os.path.sep
-        string += force_unicode(s)
+        string += s
         first = False
     return string
 

jedi/api/helpers.py

@@ -4,38 +4,50 @@ Helpers for the API
 import re
 from collections import namedtuple
 from textwrap import dedent
+from itertools import chain
 from functools import wraps
+from inspect import Parameter
 
 from parso.python.parser import Parser
 from parso.python import tree
 
-from jedi._compatibility import u, Parameter
 from jedi.inference.base_value import NO_VALUES
 from jedi.inference.syntax_tree import infer_atom
 from jedi.inference.helpers import infer_call_of_leaf
 from jedi.inference.compiled import get_string_value_set
-from jedi.cache import signature_time_cache
+from jedi.cache import signature_time_cache, memoize_method
+from jedi.parser_utils import get_parent_scope
 
 
 CompletionParts = namedtuple('CompletionParts', ['path', 'has_dot', 'name'])
 
 
-def start_match(string, like_name):
+def _start_match(string, like_name):
     return string.startswith(like_name)
 
 
-def fuzzy_match(string, like_name):
+def _fuzzy_match(string, like_name):
     if len(like_name) <= 1:
         return like_name in string
     pos = string.find(like_name[0])
     if pos >= 0:
-        return fuzzy_match(string[pos + 1:], like_name[1:])
+        return _fuzzy_match(string[pos + 1:], like_name[1:])
     return False
 
 
+def match(string, like_name, fuzzy=False):
+    if fuzzy:
+        return _fuzzy_match(string, like_name)
+    else:
+        return _start_match(string, like_name)
+
+
 def sorted_definitions(defs):
     # Note: `or ''` below is required because `module_path` could be
-    return sorted(defs, key=lambda x: (x.module_path or '', x.line or 0, x.column or 0, x.name))
+    return sorted(defs, key=lambda x: (str(x.module_path or ''),
+                                       x.line or 0,
+                                       x.column or 0,
+                                       x.name))
 
 
 def get_on_completion_name(module_node, lines, position):
@@ -75,18 +87,18 @@ def _get_code_for_stack(code_lines, leaf, position):
         # If we're not on a comment simply get the previous leaf and proceed.
         leaf = leaf.get_previous_leaf()
         if leaf is None:
-            return u('')  # At the beginning of the file.
+            return ''  # At the beginning of the file.
 
     is_after_newline = leaf.type == 'newline'
     while leaf.type == 'newline':
         leaf = leaf.get_previous_leaf()
         if leaf is None:
-            return u('')
+            return ''
 
     if leaf.type == 'error_leaf' or leaf.type == 'string':
         if leaf.start_pos[0] < position[0]:
             # On a different line, we just begin anew.
-            return u('')
+            return ''
 
         # Error leafs cannot be parsed, completion in strings is also
         # impossible.
@@ -102,7 +114,7 @@ def _get_code_for_stack(code_lines, leaf, position):
             if user_stmt.start_pos[1] > position[1]:
                 # This means that it's actually a dedent and that means that we
                 # start without value (part of a suite).
-                return u('')
+                return ''
 
         # This is basically getting the relevant lines.
         return _get_code(code_lines, user_stmt.get_start_pos_of_prefix(), position)
@@ -186,15 +198,13 @@ def filter_follow_imports(names, follow_builtin_imports=False):
             if found_builtin:
                 yield name
             else:
-                for new_name in new_names:
-                    yield new_name
+                yield from new_names
         else:
             yield name
 
 
-class CallDetails(object):
+class CallDetails:
     def __init__(self, bracket_leaf, children, position):
-        ['bracket_leaf', 'call_index', 'keyword_name_str']
         self.bracket_leaf = bracket_leaf
         self._children = children
         self._position = position
@@ -207,11 +217,15 @@ class CallDetails(object):
     def keyword_name_str(self):
         return _get_index_and_key(self._children, self._position)[1]
 
+    @memoize_method
+    def _list_arguments(self):
+        return list(_iter_arguments(self._children, self._position))
+
     def calculate_index(self, param_names):
         positional_count = 0
         used_names = set()
         star_count = -1
-        args = list(_iter_arguments(self._children, self._position))
+        args = self._list_arguments()
         if not args:
             if param_names:
                 return 0
@@ -258,6 +272,19 @@ class CallDetails(object):
                     return i
         return None
 
+    def iter_used_keyword_arguments(self):
+        for star_count, key_start, had_equal in list(self._list_arguments()):
+            if had_equal and key_start:
+                yield key_start
+
+    def count_positional_arguments(self):
+        count = 0
+        for star_count, key_start, had_equal in self._list_arguments()[:-1]:
+            if star_count or key_start:
+                break
+            count += 1
+        return count
+
 
 def _iter_arguments(nodes, position):
     def remove_after_pos(name):
@@ -268,8 +295,7 @@ def _iter_arguments(nodes, position):
     # Returns Generator[Tuple[star_count, Optional[key_start: str], had_equal]]
     nodes_before = [c for c in nodes if c.start_pos < position]
     if nodes_before[-1].type == 'arglist':
-        for x in _iter_arguments(nodes_before[-1].children, position):
-            yield x  # Python 2 :(
+        yield from _iter_arguments(nodes_before[-1].children, position)
         return
 
     previous_node_yielded = False
@@ -280,7 +306,7 @@ def _iter_arguments(nodes, position):
             first = node.children[0]
             second = node.children[1]
             if second == '=':
-                if second.start_pos < position:
+                if second.start_pos < position and first.type == 'name':
                     yield 0, first.value, True
                 else:
                     yield 0, remove_after_pos(first), False
@@ -294,7 +320,7 @@ def _iter_arguments(nodes, position):
                 else:
                     yield 0, None, False
             stars_seen = 0
-        elif node.type in ('testlist', 'testlist_star_expr'):  # testlist is Python 2
+        elif node.type == 'testlist_star_expr':
             for n in node.children[::2]:
                 if n.type == 'star_expr':
                     stars_seen = 1
@@ -376,7 +402,7 @@ def get_signature_details(module, position):
     # parents for possible function definitions.
     node = leaf.parent
     while node is not None:
-        if node.type in ('funcdef', 'classdef'):
+        if node.type in ('funcdef', 'classdef', 'decorated', 'async_stmt'):
             # Don't show signatures if there's stuff before it that just
             # makes it feel strange to have a signature.
             return None
@@ -396,7 +422,8 @@ def get_signature_details(module, position):
                 additional_children.insert(0, n)
 
         # Find a valid trailer
-        if node.type == 'trailer' and node.children[0] == '(':
+        if node.type == 'trailer' and node.children[0] == '(' \
+                or node.type == 'decorator' and node.children[2] == '(':
             # Additionally we have to check that an ending parenthesis isn't
             # interpreted wrong. There are two cases:
             # 1. Cursor before paren -> The current signature is good
@@ -405,7 +432,11 @@ def get_signature_details(module, position):
                 leaf = node.get_previous_leaf()
                 if leaf is None:
                     return None
-                return CallDetails(node.children[0], node.children, position)
+                return CallDetails(
+                    node.children[0] if node.type == 'trailer' else node.children[2],
+                    node.children,
+                    position
+                )
 
         node = node.parent
 
@@ -444,8 +475,8 @@ def validate_line_column(func):
         line_string = self._code_lines[line - 1]
         line_len = len(line_string)
         if line_string.endswith('\r\n'):
-            line_len -= 1
-        if line_string.endswith('\n'):
+            line_len -= 2
+        elif line_string.endswith('\n'):
             line_len -= 1
 
         column = line_len if column is None else column
@@ -455,3 +486,37 @@ def validate_line_column(func):
                                  column, line_len, line, line_string))
         return func(self, line, column, *args, **kwargs)
     return wrapper
+
+
+def get_module_names(module, all_scopes, definitions=True, references=False):
+    """
+    Returns a dictionary with name parts as keys and their call paths as
+    values.
+    """
+    def def_ref_filter(name):
+        is_def = name.is_definition()
+        return definitions and is_def or references and not is_def
+
+    names = list(chain.from_iterable(module.get_used_names().values()))
+    if not all_scopes:
+        # We have to filter all the names that don't have the module as a
+        # parent_scope. There's None as a parent, because nodes in the module
+        # node have the parent module and not suite as all the others.
+        # Therefore it's important to catch that case.
+
+        def is_module_scope_name(name):
+            parent_scope = get_parent_scope(name)
+            # async functions have an extra wrapper. Strip it.
+            if parent_scope and parent_scope.type == 'async_stmt':
+                parent_scope = parent_scope.parent
+            return parent_scope in (module, None)
+
+        names = [n for n in names if is_module_scope_name(n)]
+    return filter(def_ref_filter, names)
+
+
+def split_search_string(name):
+    type, _, dotted_names = name.rpartition(' ')
+    if type == 'def':
+        type = 'function'
+    return type, dotted_names.split('.')

jedi/api/interpreter.py

@@ -3,6 +3,9 @@ TODO Some parts of this module are still not well documented.
 """
 
 from jedi.inference import compiled
+from jedi.inference.base_value import ValueSet
+from jedi.inference.filters import ParserTreeFilter, MergedFilter
+from jedi.inference.names import TreeNameDefinition
 from jedi.inference.compiled import mixed
 from jedi.inference.compiled.access import create_access_path
 from jedi.inference.context import ModuleContext
@@ -14,15 +17,42 @@ def _create(inference_state, obj):
     )
 
 
-class NamespaceObject(object):
+class NamespaceObject:
     def __init__(self, dct):
         self.__dict__ = dct
 
 
+class MixedTreeName(TreeNameDefinition):
+    def infer(self):
+        """
+        In IPython notebook it is typical that some parts of the code that is
+        provided was already executed. In that case if something is not properly
+        inferred, it should still infer from the variables it already knows.
+        """
+        inferred = super().infer()
+        if not inferred:
+            for compiled_value in self.parent_context.mixed_values:
+                for f in compiled_value.get_filters():
+                    values = ValueSet.from_sets(
+                        n.infer() for n in f.get(self.string_name)
+                    )
+                    if values:
+                        return values
+        return inferred
+
+
+class MixedParserTreeFilter(ParserTreeFilter):
+    name_class = MixedTreeName
+
+
 class MixedModuleContext(ModuleContext):
     def __init__(self, tree_module_value, namespaces):
-        super(MixedModuleContext, self).__init__(tree_module_value)
-        self._namespace_objects = [NamespaceObject(n) for n in namespaces]
+        super().__init__(tree_module_value)
+        self.mixed_values = [
+            self._get_mixed_object(
+                _create(self.inference_state, NamespaceObject(n))
+            ) for n in namespaces
+        ]
 
     def _get_mixed_object(self, compiled_value):
         return mixed.MixedObject(
@@ -30,12 +60,15 @@ class MixedModuleContext(ModuleContext):
             tree_value=self._value
         )
 
-    def get_filters(self, *args, **kwargs):
-        for filter in self._value.as_context().get_filters(*args, **kwargs):
-            yield filter
+    def get_filters(self, until_position=None, origin_scope=None):
+        yield MergedFilter(
+            MixedParserTreeFilter(
+                parent_context=self,
+                until_position=until_position,
+                origin_scope=origin_scope
+            ),
+            self.get_global_filter(),
+        )
 
-        for namespace_obj in self._namespace_objects:
-            compiled_value = _create(self.inference_state, namespace_obj)
-            mixed_object = self._get_mixed_object(compiled_value)
-            for filter in mixed_object.get_filters(*args, **kwargs):
-                yield filter
+        for mixed_object in self.mixed_values:
+            yield from mixed_object.get_filters(until_position, origin_scope)

jedi/api/keywords.py

@@ -1,22 +1,19 @@
 import pydoc
+from contextlib import suppress
+from typing import Dict, Optional
 
-from jedi.inference.utils import ignored
 from jedi.inference.names import AbstractArbitraryName
 
 try:
-    from pydoc_data import topics as pydoc_topics
+    from pydoc_data import topics
+    pydoc_topics: Optional[Dict[str, str]] = topics.topics
 except ImportError:
-    # Python 2
-    try:
-        import pydoc_topics
-    except ImportError:
-        # This is for Python 3 embeddable version, which dont have
-        # pydoc_data module in its file python3x.zip.
-        pydoc_topics = None
+    # Python 3.6.8 embeddable does not have pydoc_data.
+    pydoc_topics = None
 
 
 class KeywordName(AbstractArbitraryName):
-    api_type = u'keyword'
+    api_type = 'keyword'
 
     def py__doc__(self):
         return imitate_pydoc(self.string_name)
@@ -30,11 +27,8 @@ def imitate_pydoc(string):
     if pydoc_topics is None:
         return ''
 
-    # str needed because of possible unicode stuff in py2k (pydoc doesn't work
-    # with unicode strings)
-    string = str(string)
     h = pydoc.help
-    with ignored(KeyError):
+    with suppress(KeyError):
         # try to access symbols
         string = h.symbols[string]
         string, _, related = string.partition(' ')
@@ -52,6 +46,6 @@ def imitate_pydoc(string):
         return ''
 
     try:
-        return pydoc_topics.topics[label].strip() if pydoc_topics else ''
+        return pydoc_topics[label].strip() if pydoc_topics else ''
     except KeyError:
         return ''

jedi/api/project.py

@@ -1,21 +1,52 @@
-import os
-import json
+"""
+Projects are a way to handle Python projects within Jedi. For simpler plugins
+you might not want to deal with projects, but if you want to give the user more
+flexibility to define sys paths and Python interpreters for a project,
+:class:`.Project` is the perfect way to allow for that.
 
-from jedi._compatibility import FileNotFoundError, PermissionError, IsADirectoryError
-from jedi.api.environment import SameEnvironment, \
-    get_cached_default_environment
+Projects can be saved to disk and loaded again, to allow project definitions to
+be used across repositories.
+"""
+import json
+from pathlib import Path
+from itertools import chain
+
+from jedi import debug
+from jedi.api.environment import get_cached_default_environment, create_environment
 from jedi.api.exceptions import WrongVersion
-from jedi._compatibility import force_unicode
+from jedi.api.completion import search_in_module
+from jedi.api.helpers import split_search_string, get_module_names
+from jedi.inference.imports import load_module_from_path, \
+    load_namespace_from_path, iter_module_names
 from jedi.inference.sys_path import discover_buildout_paths
 from jedi.inference.cache import inference_state_as_method_param_cache
-from jedi.common.utils import traverse_parents
+from jedi.inference.references import recurse_find_python_folders_and_files, search_in_file_ios
+from jedi.file_io import FolderIO
 
 _CONFIG_FOLDER = '.jedi'
-_CONTAINS_POTENTIAL_PROJECT = 'setup.py', '.git', '.hg', 'requirements.txt', 'MANIFEST.in'
+_CONTAINS_POTENTIAL_PROJECT = \
+    'setup.py', '.git', '.hg', 'requirements.txt', 'MANIFEST.in', 'pyproject.toml'
 
 _SERIALIZER_VERSION = 1
 
 
+def _try_to_skip_duplicates(func):
+    def wrapper(*args, **kwargs):
+        found_tree_nodes = []
+        found_modules = []
+        for definition in func(*args, **kwargs):
+            tree_node = definition._name.tree_name
+            if tree_node is not None and tree_node in found_tree_nodes:
+                continue
+            if definition.type == 'module' and definition.module_path is not None:
+                if definition.module_path in found_modules:
+                    continue
+                found_modules.append(definition.module_path)
+            yield definition
+            found_tree_nodes.append(tree_node)
+    return wrapper
+
+
 def _remove_duplicates_from_path(path):
     used = set()
     for p in path:
@@ -25,68 +56,135 @@ def _remove_duplicates_from_path(path):
         yield p
 
 
-def _force_unicode_list(lst):
-    return list(map(force_unicode, lst))
-
-
-class Project(object):
-    # TODO serialize environment
-    _serializer_ignore_attributes = ('_environment',)
+class Project:
+    """
+    Projects are a simple way to manage Python folders and define how Jedi does
+    import resolution. It is mostly used as a parameter to :class:`.Script`.
+    Additionally there are functions to search a whole project.
+    """
     _environment = None
 
+    @staticmethod
+    def _get_config_folder_path(base_path):
+        return base_path.joinpath(_CONFIG_FOLDER)
+
     @staticmethod
     def _get_json_path(base_path):
-        return os.path.join(base_path, _CONFIG_FOLDER, 'project.json')
+        return Project._get_config_folder_path(base_path).joinpath('project.json')
 
     @classmethod
     def load(cls, path):
         """
+        Loads a project from a specific path. You should not provide the path
+        to ``.jedi/project.json``, but rather the path to the project folder.
+
         :param path: The path of the directory you want to use as a project.
         """
+        if isinstance(path, str):
+            path = Path(path)
         with open(cls._get_json_path(path)) as f:
             version, data = json.load(f)
 
         if version == 1:
-            self = cls.__new__()
-            self.__dict__.update(data)
-            return self
+            return cls(**data)
         else:
             raise WrongVersion(
                 "The Jedi version of this project seems newer than what we can handle."
             )
 
-    def __init__(self, path, **kwargs):
+    def save(self):
+        """
+        Saves the project configuration in the project in ``.jedi/project.json``.
+        """
+        data = dict(self.__dict__)
+        data.pop('_environment', None)
+        data.pop('_django', None)  # TODO make django setting public?
+        data = {k.lstrip('_'): v for k, v in data.items()}
+        data['path'] = str(data['path'])
+
+        self._get_config_folder_path(self._path).mkdir(parents=True, exist_ok=True)
+        with open(self._get_json_path(self._path), 'w') as f:
+            return json.dump((_SERIALIZER_VERSION, data), f)
+
+    def __init__(
+        self,
+        path,
+        *,
+        environment_path=None,
+        load_unsafe_extensions=False,
+        sys_path=None,
+        added_sys_path=(),
+        smart_sys_path=True,
+    ) -> None:
         """
         :param path: The base path for this project.
+        :param environment_path: The Python executable path, typically the path
+            of a virtual environment.
+        :param load_unsafe_extensions: Default False, Loads extensions that are not in the
+            sys path and in the local directories. With this option enabled,
+            this is potentially unsafe if you clone a git repository and
+            analyze it's code, because those compiled extensions will be
+            important and therefore have execution privileges.
         :param sys_path: list of str. You can override the sys path if you
-            want. By default the ``sys.path.`` is generated from the
+            want. By default the ``sys.path.`` is generated by the
             environment (virtualenvs, etc).
+        :param added_sys_path: list of str. Adds these paths at the end of the
+            sys path.
         :param smart_sys_path: If this is enabled (default), adds paths from
             local directories. Otherwise you will have to rely on your packages
             being properly configured on the ``sys.path``.
         """
-        def py2_comp(path, environment=None, sys_path=None,
-                     smart_sys_path=True, _django=False):
-            self._path = os.path.abspath(path)
-            if isinstance(environment, SameEnvironment):
-                self._environment = environment
 
-            self._sys_path = sys_path
-            self._smart_sys_path = smart_sys_path
-            self._django = _django
+        if isinstance(path, str):
+            path = Path(path).absolute()
+        self._path = path
 
-        py2_comp(path, **kwargs)
+        self._environment_path = environment_path
+        if sys_path is not None:
+            # Remap potential pathlib.Path entries
+            sys_path = list(map(str, sys_path))
+        self._sys_path = sys_path
+        self._smart_sys_path = smart_sys_path
+        self._load_unsafe_extensions = load_unsafe_extensions
+        self._django = False
+        # Remap potential pathlib.Path entries
+        self.added_sys_path = list(map(str, added_sys_path))
+        """The sys path that is going to be added at the end of the """
+
+    @property
+    def path(self):
+        """
+        The base path for this project.
+        """
+        return self._path
+
+    @property
+    def sys_path(self):
+        """
+        The sys path provided to this project. This can be None and in that
+        case will be auto generated.
+        """
+        return self._sys_path
+
+    @property
+    def smart_sys_path(self):
+        """
+        If the sys path is going to be calculated in a smart way, where
+        additional paths are added.
+        """
+        return self._smart_sys_path
+
+    @property
+    def load_unsafe_extensions(self):
+        """
+        Wheter the project loads unsafe extensions.
+        """
+        return self._load_unsafe_extensions
 
     @inference_state_as_method_param_cache()
-    def _get_base_sys_path(self, inference_state, environment=None):
-        if self._sys_path is not None:
-            return self._sys_path
-
+    def _get_base_sys_path(self, inference_state):
         # The sys path has not been set explicitly.
-        if environment is None:
-            environment = self.get_environment()
-
-        sys_path = list(environment.get_sys_path())
+        sys_path = list(inference_state.environment.get_sys_path())
         try:
             sys_path.remove('')
         except ValueError:
@@ -94,34 +192,41 @@ class Project(object):
         return sys_path
 
     @inference_state_as_method_param_cache()
-    def _get_sys_path(self, inference_state, environment=None,
-                      add_parent_paths=True, add_init_paths=False):
+    def _get_sys_path(self, inference_state, add_parent_paths=True, add_init_paths=False):
         """
         Keep this method private for all users of jedi. However internally this
         one is used like a public method.
         """
-        suffixed = []
+        suffixed = list(self.added_sys_path)
         prefixed = []
 
-        sys_path = list(self._get_base_sys_path(inference_state, environment))
+        if self._sys_path is None:
+            sys_path = list(self._get_base_sys_path(inference_state))
+        else:
+            sys_path = list(self._sys_path)
+
         if self._smart_sys_path:
-            prefixed.append(self._path)
+            prefixed.append(str(self._path))
 
             if inference_state.script_path is not None:
-                suffixed += discover_buildout_paths(inference_state, inference_state.script_path)
+                suffixed += map(str, discover_buildout_paths(
+                    inference_state,
+                    inference_state.script_path
+                ))
 
                 if add_parent_paths:
                     # Collect directories in upward search by:
                     #   1. Skipping directories with __init__.py
                     #   2. Stopping immediately when above self._path
                     traversed = []
-                    for parent_path in traverse_parents(inference_state.script_path):
-                        if not parent_path.startswith(self._path):
+                    for parent_path in inference_state.script_path.parents:
+                        if parent_path == self._path \
+                                or self._path not in parent_path.parents:
                             break
                         if not add_init_paths \
-                                and os.path.isfile(os.path.join(parent_path, "__init__.py")):
+                                and parent_path.joinpath("__init__.py").is_file():
                             continue
-                        traversed.append(parent_path)
+                        traversed.append(str(parent_path))
 
                     # AFAIK some libraries have imports like `foo.foo.bar`, which
                     # leads to the conclusion to by default prefer longer paths
@@ -129,80 +234,215 @@ class Project(object):
                     suffixed += reversed(traversed)
 
         if self._django:
-            prefixed.append(self._path)
+            prefixed.append(str(self._path))
 
         path = prefixed + sys_path + suffixed
-        return list(_force_unicode_list(_remove_duplicates_from_path(path)))
-
-    def save(self):
-        data = dict(self.__dict__)
-        for attribute in self._serializer_ignore_attributes:
-            data.pop(attribute, None)
-
-        with open(self._get_json_path(self._path), 'wb') as f:
-            return json.dump((_SERIALIZER_VERSION, data), f)
+        return list(_remove_duplicates_from_path(path))
 
     def get_environment(self):
         if self._environment is None:
-            return get_cached_default_environment()
-
+            if self._environment_path is not None:
+                self._environment = create_environment(self._environment_path, safe=False)
+            else:
+                self._environment = get_cached_default_environment()
         return self._environment
 
+    def search(self, string, *, all_scopes=False):
+        """
+        Searches a name in the whole project. If the project is very big,
+        at some point Jedi will stop searching. However it's also very much
+        recommended to not exhaust the generator. Just display the first ten
+        results to the user.
+
+        There are currently three different search patterns:
+
+        - ``foo`` to search for a definition foo in any file or a file called
+          ``foo.py`` or ``foo.pyi``.
+        - ``foo.bar`` to search for the ``foo`` and then an attribute ``bar``
+          in it.
+        - ``class foo.bar.Bar`` or ``def foo.bar.baz`` to search for a specific
+          API type.
+
+        :param bool all_scopes: Default False; searches not only for
+            definitions on the top level of a module level, but also in
+            functions and classes.
+        :yields: :class:`.Name`
+        """
+        return self._search_func(string, all_scopes=all_scopes)
+
+    def complete_search(self, string, **kwargs):
+        """
+        Like :meth:`.Script.search`, but completes that string. An empty string
+        lists all definitions in a project, so be careful with that.
+
+        :param bool all_scopes: Default False; searches not only for
+            definitions on the top level of a module level, but also in
+            functions and classes.
+        :yields: :class:`.Completion`
+        """
+        return self._search_func(string, complete=True, **kwargs)
+
+    @_try_to_skip_duplicates
+    def _search_func(self, string, complete=False, all_scopes=False):
+        # Using a Script is they easiest way to get an empty module context.
+        from jedi import Script
+        s = Script('', project=self)
+        inference_state = s._inference_state
+        empty_module_context = s._get_module_context()
+
+        debug.dbg('Search for string %s, complete=%s', string, complete)
+        wanted_type, wanted_names = split_search_string(string)
+        name = wanted_names[0]
+        stub_folder_name = name + '-stubs'
+
+        ios = recurse_find_python_folders_and_files(FolderIO(str(self._path)))
+        file_ios = []
+
+        # 1. Search for modules in the current project
+        for folder_io, file_io in ios:
+            if file_io is None:
+                file_name = folder_io.get_base_name()
+                if file_name == name or file_name == stub_folder_name:
+                    f = folder_io.get_file_io('__init__.py')
+                    try:
+                        m = load_module_from_path(inference_state, f).as_context()
+                    except FileNotFoundError:
+                        f = folder_io.get_file_io('__init__.pyi')
+                        try:
+                            m = load_module_from_path(inference_state, f).as_context()
+                        except FileNotFoundError:
+                            m = load_namespace_from_path(inference_state, folder_io).as_context()
+                else:
+                    continue
+            else:
+                file_ios.append(file_io)
+                if Path(file_io.path).name in (name + '.py', name + '.pyi'):
+                    m = load_module_from_path(inference_state, file_io).as_context()
+                else:
+                    continue
+
+            debug.dbg('Search of a specific module %s', m)
+            yield from search_in_module(
+                inference_state,
+                m,
+                names=[m.name],
+                wanted_type=wanted_type,
+                wanted_names=wanted_names,
+                complete=complete,
+                convert=True,
+                ignore_imports=True,
+            )
+
+        # 2. Search for identifiers in the project.
+        for module_context in search_in_file_ios(inference_state, file_ios,
+                                                 name, complete=complete):
+            names = get_module_names(module_context.tree_node, all_scopes=all_scopes)
+            names = [module_context.create_name(n) for n in names]
+            names = _remove_imports(names)
+            yield from search_in_module(
+                inference_state,
+                module_context,
+                names=names,
+                wanted_type=wanted_type,
+                wanted_names=wanted_names,
+                complete=complete,
+                ignore_imports=True,
+            )
+
+        # 3. Search for modules on sys.path
+        sys_path = [
+            p for p in self._get_sys_path(inference_state)
+            # Exclude the current folder which is handled by recursing the folders.
+            if p != self._path
+        ]
+        names = list(iter_module_names(inference_state, empty_module_context, sys_path))
+        yield from search_in_module(
+            inference_state,
+            empty_module_context,
+            names=names,
+            wanted_type=wanted_type,
+            wanted_names=wanted_names,
+            complete=complete,
+            convert=True,
+        )
+
     def __repr__(self):
         return '<%s: %s>' % (self.__class__.__name__, self._path)
 
 
 def _is_potential_project(path):
     for name in _CONTAINS_POTENTIAL_PROJECT:
-        if os.path.exists(os.path.join(path, name)):
-            return True
+        try:
+            if path.joinpath(name).exists():
+                return True
+        except OSError:
+            continue
     return False
 
 
 def _is_django_path(directory):
     """ Detects the path of the very well known Django library (if used) """
     try:
-        with open(os.path.join(directory, 'manage.py'), 'rb') as f:
+        with open(directory.joinpath('manage.py'), 'rb') as f:
             return b"DJANGO_SETTINGS_MODULE" in f.read()
     except (FileNotFoundError, IsADirectoryError, PermissionError):
         return False
 
-    return False
-
 
 def get_default_project(path=None):
-    if path is None:
-        path = os.getcwd()
+    """
+    If a project is not defined by the user, Jedi tries to define a project by
+    itself as well as possible. Jedi traverses folders until it finds one of
+    the following:
 
-    check = os.path.realpath(path)
+    1. A ``.jedi/config.json``
+    2. One of the following files: ``setup.py``, ``.git``, ``.hg``,
+       ``requirements.txt`` and ``MANIFEST.in``.
+    """
+    if path is None:
+        path = Path.cwd()
+    elif isinstance(path, str):
+        path = Path(path)
+
+    check = path.absolute()
     probable_path = None
     first_no_init_file = None
-    for dir in traverse_parents(check, include_current=True):
+    for dir in chain([check], check.parents):
         try:
             return Project.load(dir)
         except (FileNotFoundError, IsADirectoryError, PermissionError):
             pass
+        except NotADirectoryError:
+            continue
 
         if first_no_init_file is None:
-            if os.path.exists(os.path.join(dir, '__init__.py')):
+            if dir.joinpath('__init__.py').exists():
                 # In the case that a __init__.py exists, it's in 99% just a
                 # Python package and the project sits at least one level above.
                 continue
-            else:
+            elif not dir.is_file():
                 first_no_init_file = dir
 
         if _is_django_path(dir):
-            return Project(dir, _django=True)
+            project = Project(dir)
+            project._django = True
+            return project
 
         if probable_path is None and _is_potential_project(dir):
             probable_path = dir
 
     if probable_path is not None:
-        # TODO search for setup.py etc
         return Project(probable_path)
 
     if first_no_init_file is not None:
         return Project(first_no_init_file)
 
-    curdir = path if os.path.isdir(path) else os.path.dirname(path)
+    curdir = path if path.is_dir() else path.parent
     return Project(curdir)
+
+
+def _remove_imports(names):
+    return [
+        n for n in names
+        if n.tree_name is None or n.api_type not in ('module', 'namespace')
+    ]

jedi/api/refactoring/__init__.py

@@ -0,0 +1,264 @@
+import difflib
+from pathlib import Path
+from typing import Dict, Iterable, Tuple
+
+from parso import split_lines
+
+from jedi.api.exceptions import RefactoringError
+from jedi.inference.value.namespace import ImplicitNSName
+
+EXPRESSION_PARTS = (
+    'or_test and_test not_test comparison '
+    'expr xor_expr and_expr shift_expr arith_expr term factor power atom_expr'
+).split()
+
+
+class ChangedFile:
+    def __init__(self, inference_state, from_path, to_path,
+                 module_node, node_to_str_map):
+        self._inference_state = inference_state
+        self._from_path = from_path
+        self._to_path = to_path
+        self._module_node = module_node
+        self._node_to_str_map = node_to_str_map
+
+    def get_diff(self):
+        old_lines = split_lines(self._module_node.get_code(), keepends=True)
+        new_lines = split_lines(self.get_new_code(), keepends=True)
+
+        # Add a newline at the end if it's missing. Otherwise the diff will be
+        # very weird. A `diff -u file1 file2` would show the string:
+        #
+        #     \ No newline at end of file
+        #
+        # This is not necessary IMO, because Jedi does not really play with
+        # newlines and the ending newline does not really matter in Python
+        # files. ~dave
+        if old_lines[-1] != '':
+            old_lines[-1] += '\n'
+        if new_lines[-1] != '':
+            new_lines[-1] += '\n'
+
+        project_path = self._inference_state.project.path
+        if self._from_path is None:
+            from_p = ''
+        else:
+            try:
+                from_p = self._from_path.relative_to(project_path)
+            except ValueError:  # Happens it the path is not on th project_path
+                from_p = self._from_path
+        if self._to_path is None:
+            to_p = ''
+        else:
+            try:
+                to_p = self._to_path.relative_to(project_path)
+            except ValueError:
+                to_p = self._to_path
+        diff = difflib.unified_diff(
+            old_lines, new_lines,
+            fromfile=str(from_p),
+            tofile=str(to_p),
+        )
+        # Apparently there's a space at the end of the diff - for whatever
+        # reason.
+        return ''.join(diff).rstrip(' ')
+
+    def get_new_code(self):
+        return self._inference_state.grammar.refactor(self._module_node, self._node_to_str_map)
+
+    def apply(self):
+        if self._from_path is None:
+            raise RefactoringError(
+                'Cannot apply a refactoring on a Script with path=None'
+            )
+
+        with open(self._from_path, 'w', newline='') as f:
+            f.write(self.get_new_code())
+
+    def __repr__(self):
+        return '<%s: %s>' % (self.__class__.__name__, self._from_path)
+
+
+class Refactoring:
+    def __init__(self, inference_state, file_to_node_changes, renames=()):
+        self._inference_state = inference_state
+        self._renames = renames
+        self._file_to_node_changes = file_to_node_changes
+
+    def get_changed_files(self) -> Dict[Path, ChangedFile]:
+        def calculate_to_path(p):
+            if p is None:
+                return p
+            p = str(p)
+            for from_, to in renames:
+                if p.startswith(str(from_)):
+                    p = str(to) + p[len(str(from_)):]
+            return Path(p)
+
+        renames = self.get_renames()
+        return {
+            path: ChangedFile(
+                self._inference_state,
+                from_path=path,
+                to_path=calculate_to_path(path),
+                module_node=next(iter(map_)).get_root_node(),
+                node_to_str_map=map_
+            )
+            # We need to use `or`, because the path can be None
+            for path, map_ in sorted(
+                self._file_to_node_changes.items(),
+                key=lambda x: x[0] or Path("")
+            )
+        }
+
+    def get_renames(self) -> Iterable[Tuple[Path, Path]]:
+        """
+        Files can be renamed in a refactoring.
+        """
+        return sorted(self._renames)
+
+    def get_diff(self):
+        text = ''
+        project_path = self._inference_state.project.path
+        for from_, to in self.get_renames():
+            text += 'rename from %s\nrename to %s\n' \
+                % (_try_relative_to(from_, project_path), _try_relative_to(to, project_path))
+
+        return text + ''.join(f.get_diff() for f in self.get_changed_files().values())
+
+    def apply(self):
+        """
+        Applies the whole refactoring to the files, which includes renames.
+        """
+        for f in self.get_changed_files().values():
+            f.apply()
+
+        for old, new in self.get_renames():
+            old.rename(new)
+
+
+def _calculate_rename(path, new_name):
+    dir_ = path.parent
+    if path.name in ('__init__.py', '__init__.pyi'):
+        return dir_, dir_.parent.joinpath(new_name)
+    return path, dir_.joinpath(new_name + path.suffix)
+
+
+def rename(inference_state, definitions, new_name):
+    file_renames = set()
+    file_tree_name_map = {}
+
+    if not definitions:
+        raise RefactoringError("There is no name under the cursor")
+
+    for d in definitions:
+        # This private access is ok in a way. It's not public to
+        # protect Jedi users from seeing it.
+        tree_name = d._name.tree_name
+        if d.type == 'module' and tree_name is None and d.module_path is not None:
+            p = Path(d.module_path)
+            file_renames.add(_calculate_rename(p, new_name))
+        elif isinstance(d._name, ImplicitNSName):
+            for p in d._name._value.py__path__():
+                file_renames.add(_calculate_rename(Path(p), new_name))
+        else:
+            if tree_name is not None:
+                fmap = file_tree_name_map.setdefault(d.module_path, {})
+                fmap[tree_name] = tree_name.prefix + new_name
+    return Refactoring(inference_state, file_tree_name_map, file_renames)
+
+
+def inline(inference_state, names):
+    if not names:
+        raise RefactoringError("There is no name under the cursor")
+    if any(n.api_type in ('module', 'namespace') for n in names):
+        raise RefactoringError("Cannot inline imports, modules or namespaces")
+    if any(n.tree_name is None for n in names):
+        raise RefactoringError("Cannot inline builtins/extensions")
+
+    definitions = [n for n in names if n.tree_name.is_definition()]
+    if len(definitions) == 0:
+        raise RefactoringError("No definition found to inline")
+    if len(definitions) > 1:
+        raise RefactoringError("Cannot inline a name with multiple definitions")
+    if len(names) == 1:
+        raise RefactoringError("There are no references to this name")
+
+    tree_name = definitions[0].tree_name
+
+    expr_stmt = tree_name.get_definition()
+    if expr_stmt.type != 'expr_stmt':
+        type_ = dict(
+            funcdef='function',
+            classdef='class',
+        ).get(expr_stmt.type, expr_stmt.type)
+        raise RefactoringError("Cannot inline a %s" % type_)
+
+    if len(expr_stmt.get_defined_names(include_setitem=True)) > 1:
+        raise RefactoringError("Cannot inline a statement with multiple definitions")
+    first_child = expr_stmt.children[1]
+    if first_child.type == 'annassign' and len(first_child.children) == 4:
+        first_child = first_child.children[2]
+    if first_child != '=':
+        if first_child.type == 'annassign':
+            raise RefactoringError(
+                'Cannot inline a statement that is defined by an annotation'
+            )
+        else:
+            raise RefactoringError(
+                'Cannot inline a statement with "%s"'
+                % first_child.get_code(include_prefix=False)
+            )
+
+    rhs = expr_stmt.get_rhs()
+    replace_code = rhs.get_code(include_prefix=False)
+
+    references = [n for n in names if not n.tree_name.is_definition()]
+    file_to_node_changes = {}
+    for name in references:
+        tree_name = name.tree_name
+        path = name.get_root_context().py__file__()
+        s = replace_code
+        if rhs.type == 'testlist_star_expr' \
+                or tree_name.parent.type in EXPRESSION_PARTS \
+                or tree_name.parent.type == 'trailer' \
+                and tree_name.parent.get_next_sibling() is not None:
+            s = '(' + replace_code + ')'
+
+        of_path = file_to_node_changes.setdefault(path, {})
+
+        n = tree_name
+        prefix = n.prefix
+        par = n.parent
+        if par.type == 'trailer' and par.children[0] == '.':
+            prefix = par.parent.children[0].prefix
+            n = par
+            for some_node in par.parent.children[:par.parent.children.index(par)]:
+                of_path[some_node] = ''
+        of_path[n] = prefix + s
+
+    path = definitions[0].get_root_context().py__file__()
+    changes = file_to_node_changes.setdefault(path, {})
+    changes[expr_stmt] = _remove_indent_of_prefix(expr_stmt.get_first_leaf().prefix)
+    next_leaf = expr_stmt.get_next_leaf()
+
+    # Most of the time we have to remove the newline at the end of the
+    # statement, but if there's a comment we might not need to.
+    if next_leaf.prefix.strip(' \t') == '' \
+            and (next_leaf.type == 'newline' or next_leaf == ';'):
+        changes[next_leaf] = ''
+    return Refactoring(inference_state, file_to_node_changes)
+
+
+def _remove_indent_of_prefix(prefix):
+    r"""
+    Removes the last indentation of a prefix, e.g. " \n \n " becomes " \n \n".
+    """
+    return ''.join(split_lines(prefix, keepends=True)[:-1])
+
+
+def _try_relative_to(path: Path, base: Path) -> Path:
+    try:
+        return path.relative_to(base)
+    except ValueError:
+        return path

jedi/api/refactoring/extract.py

@@ -0,0 +1,386 @@
+from textwrap import dedent
+
+from parso import split_lines
+
+from jedi import debug
+from jedi.api.exceptions import RefactoringError
+from jedi.api.refactoring import Refactoring, EXPRESSION_PARTS
+from jedi.common import indent_block
+from jedi.parser_utils import function_is_classmethod, function_is_staticmethod
+
+
+_DEFINITION_SCOPES = ('suite', 'file_input')
+_VARIABLE_EXCTRACTABLE = EXPRESSION_PARTS + \
+    ('atom testlist_star_expr testlist test lambdef lambdef_nocond '
+     'keyword name number string fstring').split()
+
+
+def extract_variable(inference_state, path, module_node, name, pos, until_pos):
+    nodes = _find_nodes(module_node, pos, until_pos)
+    debug.dbg('Extracting nodes: %s', nodes)
+
+    is_expression, message = _is_expression_with_error(nodes)
+    if not is_expression:
+        raise RefactoringError(message)
+
+    generated_code = name + ' = ' + _expression_nodes_to_string(nodes)
+    file_to_node_changes = {path: _replace(nodes, name, generated_code, pos)}
+    return Refactoring(inference_state, file_to_node_changes)
+
+
+def _is_expression_with_error(nodes):
+    """
+    Returns a tuple (is_expression, error_string).
+    """
+    if any(node.type == 'name' and node.is_definition() for node in nodes):
+        return False, 'Cannot extract a name that defines something'
+
+    if nodes[0].type not in _VARIABLE_EXCTRACTABLE:
+        return False, 'Cannot extract a "%s"' % nodes[0].type
+    return True, ''
+
+
+def _find_nodes(module_node, pos, until_pos):
+    """
+    Looks up a module and tries to find the appropriate amount of nodes that
+    are in there.
+    """
+    start_node = module_node.get_leaf_for_position(pos, include_prefixes=True)
+
+    if until_pos is None:
+        if start_node.type == 'operator':
+            next_leaf = start_node.get_next_leaf()
+            if next_leaf is not None and next_leaf.start_pos == pos:
+                start_node = next_leaf
+
+        if _is_not_extractable_syntax(start_node):
+            start_node = start_node.parent
+
+        if start_node.parent.type == 'trailer':
+            start_node = start_node.parent.parent
+        while start_node.parent.type in EXPRESSION_PARTS:
+            start_node = start_node.parent
+
+        nodes = [start_node]
+    else:
+        # Get the next leaf if we are at the end of a leaf
+        if start_node.end_pos == pos:
+            next_leaf = start_node.get_next_leaf()
+            if next_leaf is not None:
+                start_node = next_leaf
+
+        # Some syntax is not exactable, just use its parent
+        if _is_not_extractable_syntax(start_node):
+            start_node = start_node.parent
+
+        # Find the end
+        end_leaf = module_node.get_leaf_for_position(until_pos, include_prefixes=True)
+        if end_leaf.start_pos > until_pos:
+            end_leaf = end_leaf.get_previous_leaf()
+            if end_leaf is None:
+                raise RefactoringError('Cannot extract anything from that')
+
+        parent_node = start_node
+        while parent_node.end_pos < end_leaf.end_pos:
+            parent_node = parent_node.parent
+
+        nodes = _remove_unwanted_expression_nodes(parent_node, pos, until_pos)
+
+    # If the user marks just a return statement, we return the expression
+    # instead of the whole statement, because the user obviously wants to
+    # extract that part.
+    if len(nodes) == 1 and start_node.type in ('return_stmt', 'yield_expr'):
+        return [nodes[0].children[1]]
+    return nodes
+
+
+def _replace(nodes, expression_replacement, extracted, pos,
+             insert_before_leaf=None, remaining_prefix=None):
+    # Now try to replace the nodes found with a variable and move the code
+    # before the current statement.
+    definition = _get_parent_definition(nodes[0])
+    if insert_before_leaf is None:
+        insert_before_leaf = definition.get_first_leaf()
+    first_node_leaf = nodes[0].get_first_leaf()
+
+    lines = split_lines(insert_before_leaf.prefix, keepends=True)
+    if first_node_leaf is insert_before_leaf:
+        if remaining_prefix is not None:
+            # The remaining prefix has already been calculated.
+            lines[:-1] = remaining_prefix
+    lines[-1:-1] = [indent_block(extracted, lines[-1]) + '\n']
+    extracted_prefix = ''.join(lines)
+
+    replacement_dct = {}
+    if first_node_leaf is insert_before_leaf:
+        replacement_dct[nodes[0]] = extracted_prefix + expression_replacement
+    else:
+        if remaining_prefix is None:
+            p = first_node_leaf.prefix
+        else:
+            p = remaining_prefix + _get_indentation(nodes[0])
+        replacement_dct[nodes[0]] = p + expression_replacement
+        replacement_dct[insert_before_leaf] = extracted_prefix + insert_before_leaf.value
+
+    for node in nodes[1:]:
+        replacement_dct[node] = ''
+    return replacement_dct
+
+
+def _expression_nodes_to_string(nodes):
+    return ''.join(n.get_code(include_prefix=i != 0) for i, n in enumerate(nodes))
+
+
+def _suite_nodes_to_string(nodes, pos):
+    n = nodes[0]
+    prefix, part_of_code = _split_prefix_at(n.get_first_leaf(), pos[0] - 1)
+    code = part_of_code + n.get_code(include_prefix=False) \
+        + ''.join(n.get_code() for n in nodes[1:])
+    return prefix, code
+
+
+def _split_prefix_at(leaf, until_line):
+    """
+    Returns a tuple of the leaf's prefix, split at the until_line
+    position.
+    """
+    # second means the second returned part
+    second_line_count = leaf.start_pos[0] - until_line
+    lines = split_lines(leaf.prefix, keepends=True)
+    return ''.join(lines[:-second_line_count]), ''.join(lines[-second_line_count:])
+
+
+def _get_indentation(node):
+    return split_lines(node.get_first_leaf().prefix)[-1]
+
+
+def _get_parent_definition(node):
+    """
+    Returns the statement where a node is defined.
+    """
+    while node is not None:
+        if node.parent.type in _DEFINITION_SCOPES:
+            return node
+        node = node.parent
+    raise NotImplementedError('We should never even get here')
+
+
+def _remove_unwanted_expression_nodes(parent_node, pos, until_pos):
+    """
+    This function makes it so for `1 * 2 + 3` you can extract `2 + 3`, even
+    though it is not part of the expression.
+    """
+    typ = parent_node.type
+    is_suite_part = typ in ('suite', 'file_input')
+    if typ in EXPRESSION_PARTS or is_suite_part:
+        nodes = parent_node.children
+        for i, n in enumerate(nodes):
+            if n.end_pos > pos:
+                start_index = i
+                if n.type == 'operator':
+                    start_index -= 1
+                break
+        for i, n in reversed(list(enumerate(nodes))):
+            if n.start_pos < until_pos:
+                end_index = i
+                if n.type == 'operator':
+                    end_index += 1
+
+                # Something like `not foo or bar` should not be cut after not
+                for n2 in nodes[i:]:
+                    if _is_not_extractable_syntax(n2):
+                        end_index += 1
+                    else:
+                        break
+                break
+        nodes = nodes[start_index:end_index + 1]
+        if not is_suite_part:
+            nodes[0:1] = _remove_unwanted_expression_nodes(nodes[0], pos, until_pos)
+            nodes[-1:] = _remove_unwanted_expression_nodes(nodes[-1], pos, until_pos)
+        return nodes
+    return [parent_node]
+
+
+def _is_not_extractable_syntax(node):
+    return node.type == 'operator' \
+        or node.type == 'keyword' and node.value not in ('None', 'True', 'False')
+
+
+def extract_function(inference_state, path, module_context, name, pos, until_pos):
+    nodes = _find_nodes(module_context.tree_node, pos, until_pos)
+    assert len(nodes)
+
+    is_expression, _ = _is_expression_with_error(nodes)
+    context = module_context.create_context(nodes[0])
+    is_bound_method = context.is_bound_method()
+    params, return_variables = list(_find_inputs_and_outputs(module_context, context, nodes))
+
+    # Find variables
+    # Is a class method / method
+    if context.is_module():
+        insert_before_leaf = None  # Leaf will be determined later
+    else:
+        node = _get_code_insertion_node(context.tree_node, is_bound_method)
+        insert_before_leaf = node.get_first_leaf()
+    if is_expression:
+        code_block = 'return ' + _expression_nodes_to_string(nodes) + '\n'
+        remaining_prefix = None
+        has_ending_return_stmt = False
+    else:
+        has_ending_return_stmt = _is_node_ending_return_stmt(nodes[-1])
+        if not has_ending_return_stmt:
+            # Find the actually used variables (of the defined ones). If none are
+            # used (e.g. if the range covers the whole function), return the last
+            # defined variable.
+            return_variables = list(_find_needed_output_variables(
+                context,
+                nodes[0].parent,
+                nodes[-1].end_pos,
+                return_variables
+            )) or [return_variables[-1]] if return_variables else []
+
+        remaining_prefix, code_block = _suite_nodes_to_string(nodes, pos)
+        after_leaf = nodes[-1].get_next_leaf()
+        first, second = _split_prefix_at(after_leaf, until_pos[0])
+        code_block += first
+
+        code_block = dedent(code_block)
+        if not has_ending_return_stmt:
+            output_var_str = ', '.join(return_variables)
+            code_block += 'return ' + output_var_str + '\n'
+
+    # Check if we have to raise RefactoringError
+    _check_for_non_extractables(nodes[:-1] if has_ending_return_stmt else nodes)
+
+    decorator = ''
+    self_param = None
+    if is_bound_method:
+        if not function_is_staticmethod(context.tree_node):
+            function_param_names = context.get_value().get_param_names()
+            if len(function_param_names):
+                self_param = function_param_names[0].string_name
+                params = [p for p in params if p != self_param]
+
+        if function_is_classmethod(context.tree_node):
+            decorator = '@classmethod\n'
+    else:
+        code_block += '\n'
+
+    function_code = '%sdef %s(%s):\n%s' % (
+        decorator,
+        name,
+        ', '.join(params if self_param is None else [self_param] + params),
+        indent_block(code_block)
+    )
+
+    function_call = '%s(%s)' % (
+        ('' if self_param is None else self_param + '.') + name,
+        ', '.join(params)
+    )
+    if is_expression:
+        replacement = function_call
+    else:
+        if has_ending_return_stmt:
+            replacement = 'return ' + function_call + '\n'
+        else:
+            replacement = output_var_str + ' = ' + function_call + '\n'
+
+    replacement_dct = _replace(nodes, replacement, function_code, pos,
+                               insert_before_leaf, remaining_prefix)
+    if not is_expression:
+        replacement_dct[after_leaf] = second + after_leaf.value
+    file_to_node_changes = {path: replacement_dct}
+    return Refactoring(inference_state, file_to_node_changes)
+
+
+def _check_for_non_extractables(nodes):
+    for n in nodes:
+        try:
+            children = n.children
+        except AttributeError:
+            if n.value == 'return':
+                raise RefactoringError(
+                    'Can only extract return statements if they are at the end.')
+            if n.value == 'yield':
+                raise RefactoringError('Cannot extract yield statements.')
+        else:
+            _check_for_non_extractables(children)
+
+
+def _is_name_input(module_context, names, first, last):
+    for name in names:
+        if name.api_type == 'param' or not name.parent_context.is_module():
+            if name.get_root_context() is not module_context:
+                return True
+            if name.start_pos is None or not (first <= name.start_pos < last):
+                return True
+    return False
+
+
+def _find_inputs_and_outputs(module_context, context, nodes):
+    first = nodes[0].start_pos
+    last = nodes[-1].end_pos
+
+    inputs = []
+    outputs = []
+    for name in _find_non_global_names(nodes):
+        if name.is_definition():
+            if name not in outputs:
+                outputs.append(name.value)
+        else:
+            if name.value not in inputs:
+                name_definitions = context.goto(name, name.start_pos)
+                if not name_definitions \
+                        or _is_name_input(module_context, name_definitions, first, last):
+                    inputs.append(name.value)
+
+    # Check if outputs are really needed:
+    return inputs, outputs
+
+
+def _find_non_global_names(nodes):
+    for node in nodes:
+        try:
+            children = node.children
+        except AttributeError:
+            if node.type == 'name':
+                yield node
+        else:
+            # We only want to check foo in foo.bar
+            if node.type == 'trailer' and node.children[0] == '.':
+                continue
+
+            yield from _find_non_global_names(children)
+
+
+def _get_code_insertion_node(node, is_bound_method):
+    if not is_bound_method or function_is_staticmethod(node):
+        while node.parent.type != 'file_input':
+            node = node.parent
+
+    while node.parent.type in ('async_funcdef', 'decorated', 'async_stmt'):
+        node = node.parent
+    return node
+
+
+def _find_needed_output_variables(context, search_node, at_least_pos, return_variables):
+    """
+    Searches everything after at_least_pos in a node and checks if any of the
+    return_variables are used in there and returns those.
+    """
+    for node in search_node.children:
+        if node.start_pos < at_least_pos:
+            continue
+
+        return_variables = set(return_variables)
+        for name in _find_non_global_names([node]):
+            if not name.is_definition() and name.value in return_variables:
+                return_variables.remove(name.value)
+                yield name.value
+
+
+def _is_node_ending_return_stmt(node):
+    t = node.type
+    if t == 'simple_stmt':
+        return _is_node_ending_return_stmt(node.children[0])
+    return t == 'return_stmt'

jedi/api/replstartup.py

@@ -9,7 +9,7 @@ just use IPython instead::
 Then you will be able to use Jedi completer in your Python interpreter::
 
     $ python
-    Python 2.7.2+ (default, Jul 20 2012, 22:15:08)
+    Python 3.9.2+ (default, Jul 20 2020, 22:15:08)
     [GCC 4.6.1] on linux2
     Type "help", "copyright", "credits" or "license" for more information.
     >>> import os

jedi/api/strings.py

@@ -9,7 +9,6 @@ names in a module, but pretty much an arbitrary string.
 """
 import re
 
-from jedi._compatibility import unicode
 from jedi.inference.names import AbstractArbitraryName
 from jedi.inference.helpers import infer_call_of_leaf
 from jedi.api.classes import Completion
@@ -19,7 +18,7 @@ _sentinel = object()
 
 
 class StringName(AbstractArbitraryName):
-    api_type = u'string'
+    api_type = 'string'
     is_value_name = False
 
 
@@ -37,8 +36,11 @@ def complete_dict(module_context, code_lines, leaf, position, string, fuzzy):
             string = cut_value_at_position(leaf, position)
 
         context = module_context.create_context(bracket_leaf)
-        before_bracket_leaf = bracket_leaf.get_previous_leaf()
-        if before_bracket_leaf.type in ('atom', 'trailer', 'name'):
+
+        before_node = before_bracket_leaf = bracket_leaf.get_previous_leaf()
+        if before_node in (')', ']', '}'):
+            before_node = before_node.parent
+        if before_node.type in ('atom', 'trailer', 'name'):
             values = infer_call_of_leaf(context, before_bracket_leaf)
             return list(_completions_for_dicts(
                 module_context.inference_state,
@@ -65,7 +67,7 @@ def _completions_for_dicts(inference_state, dicts, literal_string, cut_end_quote
 
 
 def _create_repr_string(literal_string, dict_key):
-    if not isinstance(dict_key, (unicode, bytes)) or not literal_string:
+    if not isinstance(dict_key, (str, bytes)) or not literal_string:
         return repr(dict_key)
 
     r = repr(dict_key)
@@ -93,17 +95,16 @@ def _get_string_prefix_and_quote(string):
     return match.group(1), match.group(2)
 
 
-def _get_string_quote(string):
-    return _get_string_prefix_and_quote(string)[1]
-
-
 def _matches_quote_at_position(code_lines, quote, position):
     string = code_lines[position[0] - 1][position[1]:position[1] + len(quote)]
     return string == quote
 
 
 def get_quote_ending(string, code_lines, position, invert_result=False):
-    quote = _get_string_quote(string)
+    _, quote = _get_string_prefix_and_quote(string)
+    if quote is None:
+        return ''
+
     # Add a quote only if it's not already there.
     if _matches_quote_at_position(code_lines, quote, position) != invert_result:
         return ''

jedi/cache.py

@@ -13,14 +13,15 @@ these variables are being cleaned after every API usage.
 """
 import time
 from functools import wraps
+from typing import Any, Dict, Tuple
 
 from jedi import settings
 from parso.cache import parser_cache
 
-_time_caches = {}
+_time_caches: Dict[str, Dict[Any, Tuple[float, Any]]] = {}
 
 
-def clear_time_caches(delete_all=False):
+def clear_time_caches(delete_all: bool = False) -> None:
     """ Jedi caches many things, that should be completed after each completion
     finishes.
 

jedi/common.py

@@ -1,18 +1,6 @@
-import os
 from contextlib import contextmanager
 
 
-def traverse_parents(path, include_current=False):
-    if not include_current:
-        path = os.path.dirname(path)
-
-    previous = None
-    while previous != path:
-        yield path
-        previous = path
-        path = os.path.dirname(path)
-
-
 @contextmanager
 def monkeypatch(obj, attribute_name, new_value):
     """
@@ -24,3 +12,13 @@ def monkeypatch(obj, attribute_name, new_value):
         yield
     finally:
         setattr(obj, attribute_name, old_value)
+
+
+def indent_block(text, indention='    '):
+    """This function indents a text block with a default of four spaces."""
+    temp = ''
+    while text and text[-1] == '\n':
+        temp += text[-1]
+        text = text[:-1]
+    lines = text.split('\n')
+    return '\n'.join(map(lambda s: indention + s, lines)) + temp

jedi/common/__init__.py

@@ -1 +0,0 @@
-from jedi.common.value import BaseValueSet, BaseValue

jedi/common/value.py

@@ -1,76 +0,0 @@
-class BaseValue(object):
-    def __init__(self, inference_state, parent_context=None):
-        self.inference_state = inference_state
-        self.parent_context = parent_context
-
-    def get_root_context(self):
-        value = self
-        while True:
-            if value.parent_context is None:
-                return value
-            value = value.parent_context
-
-
-class BaseValueSet(object):
-    def __init__(self, iterable):
-        self._set = frozenset(iterable)
-        for value in iterable:
-            assert not isinstance(value, BaseValueSet)
-
-    @classmethod
-    def _from_frozen_set(cls, frozenset_):
-        self = cls.__new__(cls)
-        self._set = frozenset_
-        return self
-
-    @classmethod
-    def from_sets(cls, sets):
-        """
-        Used to work with an iterable of set.
-        """
-        aggregated = set()
-        for set_ in sets:
-            if isinstance(set_, BaseValueSet):
-                aggregated |= set_._set
-            else:
-                aggregated |= frozenset(set_)
-        return cls._from_frozen_set(frozenset(aggregated))
-
-    def __or__(self, other):
-        return self._from_frozen_set(self._set | other._set)
-
-    def __and__(self, other):
-        return self._from_frozen_set(self._set & other._set)
-
-    def __iter__(self):
-        for element in self._set:
-            yield element
-
-    def __bool__(self):
-        return bool(self._set)
-
-    def __len__(self):
-        return len(self._set)
-
-    def __repr__(self):
-        return 'S{%s}' % (', '.join(str(s) for s in self._set))
-
-    def filter(self, filter_func):
-        return self.__class__(filter(filter_func, self._set))
-
-    def __getattr__(self, name):
-        def mapper(*args, **kwargs):
-            return self.from_sets(
-                getattr(value, name)(*args, **kwargs)
-                for value in self._set
-            )
-        return mapper
-
-    def __eq__(self, other):
-        return self._set == other._set
-
-    def __ne__(self, other):
-        return not self.__eq__(other)
-
-    def __hash__(self):
-        return hash(self._set)

jedi/debug.py

@@ -1,8 +1,7 @@
 import os
 import time
 from contextlib import contextmanager
-
-from jedi._compatibility import encoding, is_py3, u
+from typing import Callable, Optional
 
 _inited = False
 
@@ -22,7 +21,7 @@ try:
         raise ImportError
     else:
         # Use colorama for nicer console output.
-        from colorama import Fore, init
+        from colorama import Fore, init  # type: ignore[import]
         from colorama import initialise
 
         def _lazy_colorama_init():  # noqa: F811
@@ -47,7 +46,7 @@ try:
             _inited = True
 
 except ImportError:
-    class Fore(object):
+    class Fore:  # type: ignore[no-redef]
         RED = ''
         GREEN = ''
         YELLOW = ''
@@ -64,7 +63,7 @@ enable_warning = False
 enable_notice = False
 
 # callback, interface: level, str
-debug_function = None
+debug_function: Optional[Callable[[str, str], None]] = None
 _debug_indent = 0
 _start_time = time.time()
 
@@ -84,39 +83,34 @@ def increase_indent(func):
 
 
 @contextmanager
-def increase_indent_cm(title=None):
+def increase_indent_cm(title=None, color='MAGENTA'):
     global _debug_indent
     if title:
-        dbg('Start: ' + title, color='MAGENTA')
+        dbg('Start: ' + title, color=color)
     _debug_indent += 1
     try:
         yield
     finally:
         _debug_indent -= 1
         if title:
-            dbg('End: ' + title, color='MAGENTA')
+            dbg('End: ' + title, color=color)
 
 
-def dbg(message, *args, **kwargs):
+def dbg(message, *args, color='GREEN'):
     """ Looks at the stack, to see if a debug message should be printed. """
-    # Python 2 compatibility, because it doesn't understand default args
-    color = kwargs.pop('color', 'GREEN')
     assert color
 
     if debug_function and enable_notice:
         i = ' ' * _debug_indent
         _lazy_colorama_init()
-        debug_function(color, i + 'dbg: ' + message % tuple(u(repr(a)) for a in args))
+        debug_function(color, i + 'dbg: ' + message % tuple(repr(a) for a in args))
 
 
-def warning(message, *args, **kwargs):
-    format = kwargs.pop('format', True)
-    assert not kwargs
-
+def warning(message, *args, format=True):
     if debug_function and enable_warning:
         i = ' ' * _debug_indent
         if format:
-            message = message % tuple(u(repr(a)) for a in args)
+            message = message % tuple(repr(a) for a in args)
         debug_function('RED', i + 'warning: ' + message)
 
 
@@ -135,9 +129,4 @@ def print_to_stdout(color, str_out):
     """
     col = getattr(Fore, color)
     _lazy_colorama_init()
-    if not is_py3:
-        str_out = str_out.encode(encoding, 'replace')
     print(col + str_out + Fore.RESET)
-
-
-# debug_function = print_to_stdout

jedi/file_io.py

@@ -3,10 +3,13 @@ import os
 from parso import file_io
 
 
-class AbstractFolderIO(object):
+class AbstractFolderIO:
     def __init__(self, path):
         self.path = path
 
+    def get_base_name(self):
+        raise NotImplementedError
+
     def list(self):
         raise NotImplementedError
 
@@ -54,7 +57,7 @@ class FolderIO(AbstractFolderIO):
                     del dirs[i]
 
 
-class FileIOFolderMixin(object):
+class FileIOFolderMixin:
     def get_parent_folder(self):
         return FolderIO(os.path.dirname(self.path))
 
@@ -62,13 +65,13 @@ class FileIOFolderMixin(object):
 class ZipFileIO(file_io.KnownContentFileIO, FileIOFolderMixin):
     """For .zip and .egg archives"""
     def __init__(self, path, code, zip_path):
-        super(ZipFileIO, self).__init__(path, code)
+        super().__init__(path, code)
         self._zip_path = zip_path
 
     def get_last_modified(self):
         try:
             return os.path.getmtime(self._zip_path)
-        except OSError:  # Python 3 would probably only need FileNotFoundError
+        except (FileNotFoundError, PermissionError, NotADirectoryError):
             return None
 
 

jedi/inference/__init__.py

@@ -63,7 +63,6 @@ only *inferes* what needs to be *inferred*. All the statements and modules
 that are not used are just being ignored.
 """
 import parso
-from parso import python_bytes_to_unicode
 from jedi.file_io import FileIO
 
 from jedi import debug
@@ -82,7 +81,7 @@ from jedi.inference.imports import follow_error_node_imports_if_possible
 from jedi.plugins import plugin_manager
 
 
-class InferenceState(object):
+class InferenceState:
     def __init__(self, project, environment=None, script_path=None):
         if environment is None:
             environment = project.get_environment()
@@ -91,7 +90,7 @@ class InferenceState(object):
         self.compiled_subprocess = environment.get_inference_state_subprocess(self)
         self.grammar = environment.get_grammar()
 
-        self.latest_grammar = parso.load_grammar(version='3.7')
+        self.latest_grammar = parso.load_grammar(version='3.13')
         self.memoize_cache = {}  # for memoize decorators
         self.module_cache = imports.ModuleCache()  # does the job of `sys.modules`.
         self.stub_module_cache = {}  # Dict[Tuple[str, ...], Optional[ModuleValue]]
@@ -100,10 +99,11 @@ class InferenceState(object):
         self.mixed_cache = {}  # see `inference.compiled.mixed._create()`
         self.analysis = []
         self.dynamic_params_depth = 0
+        self.do_dynamic_params_search = settings.dynamic_params
         self.is_analysis = False
         self.project = project
         self.access_cache = {}
-        self.allow_descriptor_getattr = False
+        self.allow_unsafe_executions = False
         self.flow_analysis_enabled = True
 
         self.reset_recursion_limitations()
@@ -121,19 +121,18 @@ class InferenceState(object):
         debug.dbg('execute result: %s in %s', value_set, value)
         return value_set
 
-    @property
+    # mypy doesn't suppport decorated propeties (https://github.com/python/mypy/issues/1362)
+    @property  # type: ignore[misc]
     @inference_state_function_cache()
     def builtins_module(self):
-        module_name = u'builtins'
-        if self.environment.version_info.major == 2:
-            module_name = u'__builtin__'
-        builtins_module, = self.import_module((module_name,), sys_path=())
+        module_name = 'builtins'
+        builtins_module, = self.import_module((module_name,), sys_path=[])
         return builtins_module
 
-    @property
+    @property  # type: ignore[misc]
     @inference_state_function_cache()
     def typing_module(self):
-        typing_module, = self.import_module((u'typing',))
+        typing_module, = self.import_module(('typing',))
         return typing_module
 
     def reset_recursion_limitations(self):
@@ -142,7 +141,7 @@ class InferenceState(object):
 
     def get_sys_path(self, **kwargs):
         """Convenience function"""
-        return self.project._get_sys_path(self, environment=self.environment, **kwargs)
+        return self.project._get_sys_path(self, **kwargs)
 
     def infer(self, context, name):
         def_ = name.get_definition(import_name_always=True)
@@ -172,6 +171,8 @@ class InferenceState(object):
                 return tree_name_to_values(self, context, name)
             elif type_ == 'param':
                 return context.py__getattribute__(name.value, position=name.end_pos)
+            elif type_ == 'namedexpr_test':
+                return context.infer_node(def_)
         else:
             result = follow_error_node_imports_if_possible(context, name)
             if result is not None:
@@ -179,14 +180,14 @@ class InferenceState(object):
 
         return helpers.infer_call_of_leaf(context, name)
 
-    def parse_and_get_code(self, code=None, path=None, encoding='utf-8',
+    def parse_and_get_code(self, code=None, path=None,
                            use_latest_grammar=False, file_io=None, **kwargs):
         if code is None:
             if file_io is None:
                 file_io = FileIO(path)
             code = file_io.read()
         # We cannot just use parso, because it doesn't use errors='replace'.
-        code = python_bytes_to_unicode(code, encoding=encoding, errors='replace')
+        code = parso.python_bytes_to_unicode(code, encoding='utf-8', errors='replace')
 
         if len(code) > settings._cropped_file_size:
             code = code[:settings._cropped_file_size]

jedi/inference/analysis.py

@@ -3,7 +3,6 @@ Module for statical analysis.
 """
 from parso.python import tree
 
-from jedi._compatibility import force_unicode
 from jedi import debug
 from jedi.inference.helpers import is_string
 
@@ -27,7 +26,7 @@ CODES = {
 }
 
 
-class Error(object):
+class Error:
     def __init__(self, name, module_path, start_pos, message=None):
         self.path = module_path
         self._start_pos = start_pos
@@ -50,13 +49,10 @@ class Error(object):
         first = self.__class__.__name__[0]
         return first + str(CODES[self.name][0])
 
-    def __unicode__(self):
+    def __str__(self):
         return '%s:%s:%s: %s %s' % (self.path, self.line, self.column,
                                     self.code, self.message)
 
-    def __str__(self):
-        return self.__unicode__()
-
     def __eq__(self, other):
         return (self.path == other.path and self.name == other.name
                 and self._start_pos == other._start_pos)
@@ -193,7 +189,7 @@ def _check_for_exception_catch(node_context, jedi_name, exception, payload=None)
             key, lazy_value = unpacked_args[1]
             names = list(lazy_value.infer())
             assert len(names) == 1 and is_string(names[0])
-            assert force_unicode(names[0].get_safe_value()) == payload[1].value
+            assert names[0].get_safe_value() == payload[1].value
 
             # Check objects
             key, lazy_value = unpacked_args[0]

jedi/inference/arguments.py

@@ -1,8 +1,8 @@
 import re
+from itertools import zip_longest
 
 from parso.python import tree
 
-from jedi._compatibility import zip_longest
 from jedi import debug
 from jedi.inference.utils import PushBackIterator
 from jedi.inference import analysis
@@ -35,7 +35,7 @@ class ParamIssue(Exception):
     pass
 
 
-def repack_with_argument_clinic(string, keep_arguments_param=False, keep_callback_param=False):
+def repack_with_argument_clinic(clinic_string):
     """
     Transforms a function or method with arguments to the signature that is
     given as an argument clinic notation.
@@ -46,35 +46,29 @@ def repack_with_argument_clinic(string, keep_arguments_param=False, keep_callbac
         str.split.__text_signature__
         # Results in: '($self, /, sep=None, maxsplit=-1)'
     """
-    clinic_args = list(_parse_argument_clinic(string))
-
     def decorator(func):
-        def wrapper(context, *args, **kwargs):
-            if keep_arguments_param:
-                arguments = kwargs['arguments']
-            else:
-                arguments = kwargs.pop('arguments')
-            if not keep_arguments_param:
-                kwargs.pop('callback', None)
+        def wrapper(value, arguments):
             try:
-                args += tuple(_iterate_argument_clinic(
-                    context.inference_state,
+                args = tuple(iterate_argument_clinic(
+                    value.inference_state,
                     arguments,
-                    clinic_args
+                    clinic_string,
                 ))
             except ParamIssue:
                 return NO_VALUES
             else:
-                return func(context, *args, **kwargs)
+                return func(value, *args)
 
         return wrapper
     return decorator
 
 
-def _iterate_argument_clinic(inference_state, arguments, parameters):
+def iterate_argument_clinic(inference_state, arguments, clinic_string):
     """Uses a list with argument clinic information (see PEP 436)."""
+    clinic_args = list(_parse_argument_clinic(clinic_string))
+
     iterator = PushBackIterator(arguments.unpack())
-    for i, (name, optional, allow_kwargs, stars) in enumerate(parameters):
+    for i, (name, optional, allow_kwargs, stars) in enumerate(clinic_args):
         if stars == 1:
             lazy_values = []
             for key, argument in iterator:
@@ -94,7 +88,7 @@ def _iterate_argument_clinic(inference_state, arguments, parameters):
             raise ParamIssue
         if argument is None and not optional:
             debug.warning('TypeError: %s expected at least %s arguments, got %s',
-                          name, len(parameters), i)
+                          name, len(clinic_args), i)
             raise ParamIssue
 
         value_set = NO_VALUES if argument is None else argument.infer()
@@ -130,16 +124,7 @@ def _parse_argument_clinic(string):
             allow_kwargs = True
 
 
-class _AbstractArgumentsMixin(object):
-    def infer_all(self, funcdef=None):
-        """
-        Inferes all arguments as a support for static analysis
-        (normally Jedi).
-        """
-        for key, lazy_value in self.unpack():
-            types = lazy_value.infer()
-            try_iter_content(types)
-
+class _AbstractArgumentsMixin:
     def unpack(self, funcdef=None):
         raise NotImplementedError
 
@@ -157,11 +142,8 @@ def unpack_arglist(arglist):
     if arglist is None:
         return
 
-    # Allow testlist here as well for Python2's class inheritance
-    # definitions.
-    if not (arglist.type in ('arglist', 'testlist') or (
-            # in python 3.5 **arg is an argument, not arglist
-            arglist.type == 'argument' and arglist.children[0] in ('*', '**'))):
+    if arglist.type != 'arglist' and not (
+            arglist.type == 'argument' and arglist.children[0] in ('*', '**')):
         yield 0, arglist
         return
 
@@ -170,7 +152,9 @@ def unpack_arglist(arglist):
         if child == ',':
             continue
         elif child in ('*', '**'):
-            yield len(child.value), next(iterator)
+            c = next(iterator, None)
+            assert c is not None
+            yield len(child.value), c
         elif child.type == 'argument' and \
                 child.children[0] in ('*', '**'):
             assert len(child.children) == 2
@@ -202,16 +186,13 @@ class TreeArguments(AbstractArguments):
                 iterators = [_iterate_star_args(self.context, a, el, funcdef)
                              for a in arrays]
                 for values in list(zip_longest(*iterators)):
-                    # TODO zip_longest yields None, that means this would raise
-                    # an exception?
                     yield None, get_merged_lazy_value(
                         [v for v in values if v is not None]
                     )
             elif star_count == 2:
                 arrays = self.context.infer_node(el)
                 for dct in arrays:
-                    for key, values in _star_star_dict(self.context, dct, el, funcdef):
-                        yield key, values
+                    yield from _star_star_dict(self.context, dct, el, funcdef)
             else:
                 if el.type == 'argument':
                     c = el.children
@@ -234,8 +215,7 @@ class TreeArguments(AbstractArguments):
 
         # Reordering arguments is necessary, because star args sometimes appear
         # after named argument, but in the actual order it's prepended.
-        for named_arg in named_args:
-            yield named_arg
+        yield from named_args
 
     def _as_tree_tuple_objects(self):
         for star_count, argument in unpack_arglist(self.argument_node):
@@ -336,8 +316,7 @@ def _iterate_star_args(context, array, input_node, funcdef=None):
     except AttributeError:
         pass
     else:
-        for lazy_value in iter_():
-            yield lazy_value
+        yield from iter_()
 
 
 def _star_star_dict(context, array, input_node, funcdef):

jedi/inference/base_value.py

@@ -8,12 +8,12 @@ just one.
 """
 from functools import reduce
 from operator import add
+from itertools import zip_longest
+
 from parso.python.tree import Name
 
 from jedi import debug
-from jedi._compatibility import zip_longest, unicode
 from jedi.parser_utils import clean_scope_docstring
-from jedi.common import BaseValueSet, BaseValue
 from jedi.inference.helpers import SimpleGetItemNotFound
 from jedi.inference.utils import safe_property
 from jedi.inference.cache import inference_state_as_method_param_cache
@@ -22,7 +22,11 @@ from jedi.cache import memoize_method
 sentinel = object()
 
 
-class HelperValueMixin(object):
+class HasNoContext(Exception):
+    pass
+
+
+class HelperValueMixin:
     def get_root_context(self):
         value = self
         if value.parent_context is None:
@@ -33,11 +37,6 @@ class HelperValueMixin(object):
                 return value
             value = value.parent_context
 
-    @classmethod
-    @inference_state_as_method_param_cache()
-    def create_cached(cls, *args, **kwargs):
-        return cls(*args, **kwargs)
-
     def execute(self, arguments):
         return self.inference_state.execute(self, arguments=arguments)
 
@@ -60,18 +59,14 @@ class HelperValueMixin(object):
 
     def _get_value_filters(self, name_or_str):
         origin_scope = name_or_str if isinstance(name_or_str, Name) else None
-        for f in self.get_filters(origin_scope=origin_scope):
-            yield f
+        yield from self.get_filters(origin_scope=origin_scope)
         # This covers the case where a stub files are incomplete.
         if self.is_stub():
             from jedi.inference.gradual.conversion import convert_values
             for c in convert_values(ValueSet({self})):
-                for f in c.get_filters():
-                    yield f
+                yield from c.get_filters()
 
     def goto(self, name_or_str, name_context=None, analysis_errors=True):
-        if name_context is None:
-            name_context = self
         from jedi.inference import finder
         filters = self._get_value_filters(name_or_str)
         names = finder.filter_name(filters, name_or_str)
@@ -100,11 +95,14 @@ class HelperValueMixin(object):
         return values
 
     def py__await__(self):
-        await_value_set = self.py__getattribute__(u"__await__")
+        await_value_set = self.py__getattribute__("__await__")
         if not await_value_set:
             debug.warning('Tried to run __await__ on value %s', self)
         return await_value_set.execute_with_values()
 
+    def py__name__(self):
+        return self.name.string_name
+
     def iterate(self, contextualized_node=None, is_async=False):
         debug.dbg('iterate %s', self)
         if is_async:
@@ -117,15 +115,19 @@ class HelperValueMixin(object):
                         .py__getattribute__('__anext__').execute_with_values()
                         .py__getattribute__('__await__').execute_with_values()
                         .py__stop_iteration_returns()
-                )  # noqa
+                )  # noqa: E124
             ])
         return self.py__iter__(contextualized_node)
 
     def is_sub_class_of(self, class_value):
-        for cls in self.py__mro__():
-            if cls.is_same_class(class_value):
-                return True
-        return False
+        with debug.increase_indent_cm('subclass matching of %s <=> %s' % (self, class_value),
+                                      color='BLUE'):
+            for cls in self.py__mro__():
+                if cls.is_same_class(class_value):
+                    debug.dbg('matched subclass True', color='BLUE')
+                    return True
+            debug.dbg('matched subclass False', color='BLUE')
+            return False
 
     def is_same_class(self, class2):
         # Class matching should prefer comparisons that are not this function.
@@ -138,7 +140,7 @@ class HelperValueMixin(object):
         return self._as_context(*args, **kwargs)
 
 
-class Value(HelperValueMixin, BaseValue):
+class Value(HelperValueMixin):
     """
     To be implemented by subclasses.
     """
@@ -146,12 +148,11 @@ class Value(HelperValueMixin, BaseValue):
     # Possible values: None, tuple, list, dict and set. Here to deal with these
     # very important containers.
     array_type = None
+    api_type = 'not_defined_please_report_bug'
 
-    @property
-    def api_type(self):
-        # By default just lower name of the class. Can and should be
-        # overwritten.
-        return self.__class__.__name__.lower()
+    def __init__(self, inference_state, parent_context=None):
+        self.inference_state = inference_state
+        self.parent_context = parent_context
 
     def py__getitem__(self, index_value_set, contextualized_node):
         from jedi.inference import analysis
@@ -177,12 +178,18 @@ class Value(HelperValueMixin, BaseValue):
                 message="TypeError: '%s' object is not iterable" % self)
         return iter([])
 
+    def py__next__(self, contextualized_node=None):
+        return self.py__iter__(contextualized_node)
+
     def get_signatures(self):
         return []
 
     def is_class(self):
         return False
 
+    def is_class_mixin(self):
+        return False
+
     def is_instance(self):
         return False
 
@@ -218,7 +225,6 @@ class Value(HelperValueMixin, BaseValue):
             return ''
         else:
             return clean_scope_docstring(self.tree_node)
-        return None
 
     def get_safe_value(self, default=sentinel):
         if default is sentinel:
@@ -247,6 +253,9 @@ class Value(HelperValueMixin, BaseValue):
         debug.warning("No __get__ defined on %s", self)
         return ValueSet([self])
 
+    def py__get__on_class(self, calling_instance, instance, class_value):
+        return NotImplemented
+
     def get_qualified_names(self):
         # Returns Optional[Tuple[str, ...]]
         return None
@@ -256,13 +265,43 @@ class Value(HelperValueMixin, BaseValue):
         return self.parent_context.is_stub()
 
     def _as_context(self):
-        raise NotImplementedError('Not all values need to be converted to contexts: %s', self)
+        raise HasNoContext
 
+    @property
     def name(self):
         raise NotImplementedError
 
-    def py__name__(self):
-        return self.name.string_name
+    def get_type_hint(self, add_class_info=True):
+        return None
+
+    def infer_type_vars(self, value_set):
+        """
+        When the current instance represents a type annotation, this method
+        tries to find information about undefined type vars and returns a dict
+        from type var name to value set.
+
+        This is for example important to understand what `iter([1])` returns.
+        According to typeshed, `iter` returns an `Iterator[_T]`:
+
+            def iter(iterable: Iterable[_T]) -> Iterator[_T]: ...
+
+        This functions would generate `int` for `_T` in this case, because it
+        unpacks the `Iterable`.
+
+        Parameters
+        ----------
+
+        `self`: represents the annotation of the current parameter to infer the
+            value for. In the above example, this would initially be the
+            `Iterable[_T]` of the `iterable` parameter and then, when recursing,
+            just the `_T` generic parameter.
+
+        `value_set`: represents the actual argument passed to the parameter
+            we're inferred for, or (for recursive calls) their types. In the
+            above example this would first be the representation of the list
+            `[1]` and then, when recursing, just of `1`.
+        """
+        return {}
 
 
 def iterate_values(values, contextualized_node=None, is_async=False):
@@ -321,14 +360,14 @@ class ValueWrapper(_ValueWrapperBase):
 
 class TreeValue(Value):
     def __init__(self, inference_state, parent_context, tree_node):
-        super(TreeValue, self).__init__(inference_state, parent_context)
+        super().__init__(inference_state, parent_context)
         self.tree_node = tree_node
 
     def __repr__(self):
         return '<%s: %s>' % (self.__class__.__name__, self.tree_node)
 
 
-class ContextualizedNode(object):
+class ContextualizedNode:
     def __init__(self, context, node):
         self.context = context
         self.node = node
@@ -349,7 +388,7 @@ def _getitem(value, index_values, contextualized_node):
     unused_values = set()
     for index_value in index_values:
         index = index_value.get_safe_value(default=None)
-        if type(index) in (float, int, str, unicode, slice, bytes):
+        if type(index) in (float, int, str, slice, bytes):
             try:
                 result |= value.py__simple_getitem__(index)
                 continue
@@ -370,7 +409,69 @@ def _getitem(value, index_values, contextualized_node):
     return result
 
 
-class ValueSet(BaseValueSet):
+class ValueSet:
+    def __init__(self, iterable):
+        self._set = frozenset(iterable)
+        for value in iterable:
+            assert not isinstance(value, ValueSet)
+
+    @classmethod
+    def _from_frozen_set(cls, frozenset_):
+        self = cls.__new__(cls)
+        self._set = frozenset_
+        return self
+
+    @classmethod
+    def from_sets(cls, sets):
+        """
+        Used to work with an iterable of set.
+        """
+        aggregated = set()
+        for set_ in sets:
+            if isinstance(set_, ValueSet):
+                aggregated |= set_._set
+            else:
+                aggregated |= frozenset(set_)
+        return cls._from_frozen_set(frozenset(aggregated))
+
+    def __or__(self, other):
+        return self._from_frozen_set(self._set | other._set)
+
+    def __and__(self, other):
+        return self._from_frozen_set(self._set & other._set)
+
+    def __iter__(self):
+        return iter(self._set)
+
+    def __bool__(self):
+        return bool(self._set)
+
+    def __len__(self):
+        return len(self._set)
+
+    def __repr__(self):
+        return 'S{%s}' % (', '.join(str(s) for s in self._set))
+
+    def filter(self, filter_func):
+        return self.__class__(filter(filter_func, self._set))
+
+    def __getattr__(self, name):
+        def mapper(*args, **kwargs):
+            return self.from_sets(
+                getattr(value, name)(*args, **kwargs)
+                for value in self._set
+            )
+        return mapper
+
+    def __eq__(self, other):
+        return self._set == other._set
+
+    def __ne__(self, other):
+        return not self.__eq__(other)
+
+    def __hash__(self):
+        return hash(self._set)
+
     def py__class__(self):
         return ValueSet(c.py__class__() for c in self._set)
 
@@ -414,6 +515,38 @@ class ValueSet(BaseValueSet):
     def get_signatures(self):
         return [sig for c in self._set for sig in c.get_signatures()]
 
+    def get_type_hint(self, add_class_info=True):
+        t = [v.get_type_hint(add_class_info=add_class_info) for v in self._set]
+        type_hints = sorted(filter(None, t))
+        if len(type_hints) == 1:
+            return type_hints[0]
+
+        optional = 'None' in type_hints
+        if optional:
+            type_hints.remove('None')
+
+        if len(type_hints) == 0:
+            return None
+        elif len(type_hints) == 1:
+            s = type_hints[0]
+        else:
+            s = 'Union[%s]' % ', '.join(type_hints)
+        if optional:
+            s = 'Optional[%s]' % s
+        return s
+
+    def infer_type_vars(self, value_set):
+        # Circular
+        from jedi.inference.gradual.annotation import merge_type_var_dicts
+
+        type_var_dict = {}
+        for value in self._set:
+            merge_type_var_dicts(
+                type_var_dict,
+                value.infer_type_vars(value_set),
+            )
+        return type_var_dict
+
 
 NO_VALUES = ValueSet([])
 

jedi/inference/cache.py

@@ -3,6 +3,7 @@
   default otherwise.
 - ``CachedMetaClass`` uses ``_memoize_default`` to do the same with classes.
 """
+from functools import wraps
 
 from jedi import debug
 
@@ -77,7 +78,7 @@ class CachedMetaClass(type):
     """
     @inference_state_as_method_param_cache()
     def __call__(self, *args, **kwargs):
-        return super(CachedMetaClass, self).__call__(*args, **kwargs)
+        return super().__call__(*args, **kwargs)
 
 
 def inference_state_method_generator_cache():
@@ -86,6 +87,7 @@ def inference_state_method_generator_cache():
     recursion errors and returns no further iterator elemends in that case.
     """
     def func(function):
+        @wraps(function)
         def wrapper(obj, *args, **kwargs):
             cache = obj.inference_state.memoize_cache
             try:

jedi/inference/compiled/__init__.py

@@ -1,4 +1,6 @@
-from jedi._compatibility import unicode
+# This file also re-exports symbols for wider use. We configure mypy and flake8
+# to be aware that this file does this.
+
 from jedi.inference.compiled.value import CompiledValue, CompiledName, \
     CompiledValueFilter, CompiledValueName, create_from_access_path
 from jedi.inference.base_value import LazyValueWrapper
@@ -29,7 +31,7 @@ class ExactValue(LazyValueWrapper):
         if name in ('get_safe_value', 'execute_operation', 'access_handle',
                     'negate', 'py__bool__', 'is_compiled'):
             return getattr(self._compiled_value, name)
-        return super(ExactValue, self).__getattribute__(name)
+        return super().__getattribute__(name)
 
     def _get_wrapped_value(self):
         instance, = builtin_from_name(
@@ -45,7 +47,7 @@ def create_simple_object(inference_state, obj):
     Only allows creations of objects that are easily picklable across Python
     versions.
     """
-    assert type(obj) in (int, float, str, bytes, unicode, slice, complex, bool), obj
+    assert type(obj) in (int, float, str, bytes, slice, complex, bool), repr(obj)
     compiled_value = create_from_access_path(
         inference_state,
         inference_state.compiled_subprocess.create_simple_object(obj)
@@ -54,7 +56,7 @@ def create_simple_object(inference_state, obj):
 
 
 def get_string_value_set(inference_state):
-    return builtin_from_name(inference_state, u'str').execute_with_values()
+    return builtin_from_name(inference_state, 'str').execute_with_values()
 
 
 def load_module(inference_state, dotted_name, **kwargs):

jedi/inference/compiled/access.py

@@ -1,17 +1,19 @@
-from __future__ import print_function
 import inspect
 import types
+import traceback
 import sys
 import operator as op
 from collections import namedtuple
 import warnings
 import re
+import builtins
+import typing
+from pathlib import Path
+from typing import Optional, Tuple
 
-from jedi._compatibility import unicode, is_py3, builtins, \
-    py_version, force_unicode
 from jedi.inference.compiled.getattr_static import getattr_static
 
-ALLOWED_GETITEM_TYPES = (str, list, tuple, unicode, bytes, bytearray, dict)
+ALLOWED_GETITEM_TYPES = (str, list, tuple, bytes, bytearray, dict)
 
 MethodDescriptorType = type(str.replace)
 # These are not considered classes and access is granted even though they have
@@ -28,22 +30,17 @@ NOT_CLASS_TYPES = (
     types.MethodType,
     types.ModuleType,
     types.TracebackType,
-    MethodDescriptorType
+    MethodDescriptorType,
+    types.MappingProxyType,
+    types.SimpleNamespace,
+    types.DynamicClassAttribute,
 )
 
-if is_py3:
-    NOT_CLASS_TYPES += (
-        types.MappingProxyType,
-        types.SimpleNamespace,
-        types.DynamicClassAttribute,
-    )
-
-
 # Those types don't exist in typing.
 MethodDescriptorType = type(str.replace)
 WrapperDescriptorType = type(set.__iter__)
 # `object.__subclasshook__` is an already executed descriptor.
-object_class_dict = type.__dict__["__dict__"].__get__(object)
+object_class_dict = type.__dict__["__dict__"].__get__(object)  # type: ignore[index]
 ClassMethodDescriptorType = type(object_class_dict['__subclasshook__'])
 
 _sentinel = object()
@@ -123,13 +120,18 @@ def load_module(inference_state, dotted_name, sys_path):
         __import__(dotted_name)
     except ImportError:
         # If a module is "corrupt" or not really a Python module or whatever.
-        print('Module %s not importable in path %s.' % (dotted_name, sys_path), file=sys.stderr)
+        warnings.warn(
+            "Module %s not importable in path %s." % (dotted_name, sys_path),
+            UserWarning,
+            stacklevel=2,
+        )
         return None
     except Exception:
         # Since __import__ pretty much makes code execution possible, just
         # catch any error here and print it.
-        import traceback
-        print("Cannot import:\n%s" % traceback.format_exc(), file=sys.stderr)
+        warnings.warn(
+            "Cannot import:\n%s" % traceback.format_exc(), UserWarning, stacklevel=2
+        )
         return None
     finally:
         sys.path = temp
@@ -140,42 +142,29 @@ def load_module(inference_state, dotted_name, sys_path):
     return create_access_path(inference_state, module)
 
 
-class AccessPath(object):
+class AccessPath:
     def __init__(self, accesses):
         self.accesses = accesses
 
-    # Writing both of these methods here looks a bit ridiculous. However with
-    # the differences of Python 2/3 it's actually necessary, because we will
-    # otherwise have a accesses attribute that is bytes instead of unicode.
-    def __getstate__(self):
-        return self.accesses
 
-    def __setstate__(self, value):
-        self.accesses = value
-
-
-def create_access_path(inference_state, obj):
+def create_access_path(inference_state, obj) -> AccessPath:
     access = create_access(inference_state, obj)
     return AccessPath(access.get_access_path_tuples())
 
 
-def _force_unicode_decorator(func):
-    return lambda *args, **kwargs: force_unicode(func(*args, **kwargs))
-
-
 def get_api_type(obj):
     if inspect.isclass(obj):
-        return u'class'
+        return 'class'
     elif inspect.ismodule(obj):
-        return u'module'
+        return 'module'
     elif inspect.isbuiltin(obj) or inspect.ismethod(obj) \
             or inspect.ismethoddescriptor(obj) or inspect.isfunction(obj):
-        return u'function'
+        return 'function'
     # Everything else...
-    return u'instance'
+    return 'instance'
 
 
-class DirectObjectAccess(object):
+class DirectObjectAccess:
     def __init__(self, inference_state, obj):
         self._inference_state = inference_state
         self._obj = obj
@@ -186,20 +175,20 @@ class DirectObjectAccess(object):
     def _create_access(self, obj):
         return create_access(self._inference_state, obj)
 
-    def _create_access_path(self, obj):
+    def _create_access_path(self, obj) -> AccessPath:
         return create_access_path(self._inference_state, obj)
 
     def py__bool__(self):
         return bool(self._obj)
 
-    def py__file__(self):
+    def py__file__(self) -> Optional[Path]:
         try:
-            return self._obj.__file__
+            return Path(self._obj.__file__)
         except AttributeError:
             return None
 
     def py__doc__(self):
-        return force_unicode(inspect.getdoc(self._obj)) or u''
+        return inspect.getdoc(self._obj) or ''
 
     def py__name__(self):
         if not _is_class_instance(self._obj) or \
@@ -214,7 +203,7 @@ class DirectObjectAccess(object):
                 return None
 
         try:
-            return force_unicode(cls.__name__)
+            return cls.__name__
         except AttributeError:
             return None
 
@@ -224,18 +213,39 @@ class DirectObjectAccess(object):
     def py__getitem__all_values(self):
         if isinstance(self._obj, dict):
             return [self._create_access_path(v) for v in self._obj.values()]
-        return self.py__iter__list()
+        if isinstance(self._obj, (list, tuple)):
+            return [self._create_access_path(v) for v in self._obj]
 
-    def py__simple_getitem__(self, index):
-        if type(self._obj) not in ALLOWED_GETITEM_TYPES:
+        if self.is_instance():
+            cls = DirectObjectAccess(self._inference_state, self._obj.__class__)
+            return cls.py__getitem__all_values()
+
+        try:
+            getitem = self._obj.__getitem__
+        except AttributeError:
+            pass
+        else:
+            annotation = DirectObjectAccess(self._inference_state, getitem).get_return_annotation()
+            if annotation is not None:
+                return [annotation]
+        return None
+
+    def py__simple_getitem__(self, index, *, safe=True):
+        if safe and type(self._obj) not in ALLOWED_GETITEM_TYPES:
             # Get rid of side effects, we won't call custom `__getitem__`s.
             return None
 
         return self._create_access_path(self._obj[index])
 
     def py__iter__list(self):
-        if not hasattr(self._obj, '__getitem__'):
+        try:
+            iter_method = self._obj.__iter__
+        except AttributeError:
             return None
+        else:
+            p = DirectObjectAccess(self._inference_state, iter_method).get_return_annotation()
+            if p is not None:
+                return [p]
 
         if type(self._obj) not in ALLOWED_GETITEM_TYPES:
             # Get rid of side effects, we won't call custom `__getitem__`s.
@@ -260,26 +270,23 @@ class DirectObjectAccess(object):
         # Avoid some weird hacks that would just fail, because they cannot be
         # used by pickle.
         if not isinstance(paths, list) \
-                or not all(isinstance(p, (bytes, unicode)) for p in paths):
+                or not all(isinstance(p, str) for p in paths):
             return None
         return paths
 
-    @_force_unicode_decorator
     @shorten_repr
     def get_repr(self):
-        builtins = 'builtins', '__builtin__'
-
         if inspect.ismodule(self._obj):
             return repr(self._obj)
         # Try to avoid execution of the property.
-        if safe_getattr(self._obj, '__module__', default='') in builtins:
+        if safe_getattr(self._obj, '__module__', default='') == 'builtins':
             return repr(self._obj)
 
         type_ = type(self._obj)
         if type_ == type:
             return type.__repr__(self._obj)
 
-        if safe_getattr(type_, '__module__', default='') in builtins:
+        if safe_getattr(type_, '__module__', default='') == 'builtins':
             # Allow direct execution of repr for builtins.
             return repr(self._obj)
         return object.__repr__(self._obj)
@@ -310,10 +317,10 @@ class DirectObjectAccess(object):
             name = try_to_get_name(type(self._obj))
             if name is None:
                 return ()
-        return tuple(force_unicode(n) for n in name.split('.'))
+        return tuple(name.split('.'))
 
     def dir(self):
-        return list(map(force_unicode, dir(self._obj)))
+        return dir(self._obj)
 
     def has_iter(self):
         try:
@@ -322,33 +329,37 @@ class DirectObjectAccess(object):
         except TypeError:
             return False
 
-    def is_allowed_getattr(self, name, unsafe=False):
+    def is_allowed_getattr(self, name, safe=True) -> Tuple[bool, bool, Optional[AccessPath]]:
         # TODO this API is ugly.
-        if unsafe:
-            # Unsafe is mostly used to check for __getattr__/__getattribute__.
-            # getattr_static works for properties, but the underscore methods
-            # are just ignored (because it's safer and avoids more code
-            # execution). See also GH #1378.
-
-            # Avoid warnings, see comment in the next function.
-            with warnings.catch_warnings(record=True):
-                warnings.simplefilter("always")
-                try:
-                    return hasattr(self._obj, name), False
-                except Exception:
-                    # Obviously has an attribute (propably a property) that
-                    # gets executed, so just avoid all exceptions here.
-                    return False, False
         try:
             attr, is_get_descriptor = getattr_static(self._obj, name)
         except AttributeError:
-            return False, False
+            if not safe:
+                # Unsafe is mostly used to check for __getattr__/__getattribute__.
+                # getattr_static works for properties, but the underscore methods
+                # are just ignored (because it's safer and avoids more code
+                # execution). See also GH #1378.
+
+                # Avoid warnings, see comment in the next function.
+                with warnings.catch_warnings(record=True):
+                    warnings.simplefilter("always")
+                    try:
+                        return hasattr(self._obj, name), False, None
+                    except Exception:
+                        # Obviously has an attribute (probably a property) that
+                        # gets executed, so just avoid all exceptions here.
+                        pass
+            return False, False, None
         else:
             if is_get_descriptor and type(attr) not in ALLOWED_DESCRIPTOR_ACCESS:
+                if isinstance(attr, property):
+                    if hasattr(attr.fget, '__annotations__'):
+                        a = DirectObjectAccess(self._inference_state, attr.fget)
+                        return True, True, a.get_return_annotation()
                 # In case of descriptors that have get methods we cannot return
                 # it's value, because that would mean code execution.
-                return True, True
-        return True, False
+                return True, True, None
+        return True, False, None
 
     def getattr_paths(self, name, default=_sentinel):
         try:
@@ -377,7 +388,7 @@ class DirectObjectAccess(object):
         except AttributeError:
             pass
         else:
-            if module is not None:
+            if module is not None and isinstance(module, str):
                 try:
                     __import__(module)
                     # For some modules like _sqlite3, the __module__ for classes is
@@ -396,7 +407,7 @@ class DirectObjectAccess(object):
         return [self._create_access(module), access]
 
     def get_safe_value(self):
-        if type(self._obj) in (bool, bytes, float, int, str, unicode, slice) or self._obj is None:
+        if type(self._obj) in (bool, bytes, float, int, str, slice) or self._obj is None:
             return self._obj
         raise ValueError("Object is type %s and not simple" % type(self._obj))
 
@@ -464,9 +475,6 @@ class DirectObjectAccess(object):
         """
         Returns Tuple[Optional[str], Tuple[AccessPath, ...]]
         """
-        if sys.version_info < (3, 5):
-            return None, ()
-
         name = None
         args = ()
         if safe_getattr(self._obj, '__module__', default='') == 'typing':
@@ -484,6 +492,9 @@ class DirectObjectAccess(object):
     def needs_type_completions(self):
         return inspect.isclass(self._obj) and self._obj != type
 
+    def _annotation_to_str(self, annotation):
+        return inspect.formatannotation(annotation)
+
     def get_signature_params(self):
         return [
             SignatureParam(
@@ -493,31 +504,13 @@ class DirectObjectAccess(object):
                 default_string=repr(p.default),
                 has_annotation=p.annotation is not p.empty,
                 annotation=self._create_access_path(p.annotation),
-                annotation_string=str(p.annotation),
+                annotation_string=self._annotation_to_str(p.annotation),
                 kind_name=str(p.kind)
             ) for p in self._get_signature().parameters.values()
         ]
 
     def _get_signature(self):
         obj = self._obj
-        if py_version < 33:
-            raise ValueError("inspect.signature was introduced in 3.3")
-        if py_version == 34:
-            # In 3.4 inspect.signature are wrong for str and int. This has
-            # been fixed in 3.5. The signature of object is returned,
-            # because no signature was found for str. Here we imitate 3.5
-            # logic and just ignore the signature if the magic methods
-            # don't match object.
-            # 3.3 doesn't even have the logic and returns nothing for str
-            # and classes that inherit from object.
-            user_def = inspect._signature_get_user_defined_method
-            if (inspect.isclass(obj)
-                    and not user_def(type(obj), '__init__')
-                    and not user_def(type(obj), '__new__')
-                    and (obj.__init__ != object.__init__
-                         or obj.__new__ != object.__new__)):
-                raise ValueError
-
         try:
             return inspect.signature(obj)
         except (RuntimeError, TypeError):
@@ -526,7 +519,7 @@ class DirectObjectAccess(object):
             # the signature. In that case we just want a simple escape for now.
             raise ValueError
 
-    def get_return_annotation(self):
+    def get_return_annotation(self) -> Optional[AccessPath]:
         try:
             o = self._obj.__annotations__.get('return')
         except AttributeError:
@@ -536,15 +529,9 @@ class DirectObjectAccess(object):
             return None
 
         try:
-            # Python 2 doesn't have typing.
-            import typing
-        except ImportError:
+            o = typing.get_type_hints(self._obj).get('return')
+        except Exception:
             pass
-        else:
-            try:
-                o = typing.get_type_hints(self._obj).get('return')
-            except Exception:
-                pass
 
         return self._create_access_path(o)
 
@@ -557,7 +544,7 @@ class DirectObjectAccess(object):
         objects of an objects
         """
         tuples = dict(
-            (force_unicode(name), self.is_allowed_getattr(name))
+            (name, self.is_allowed_getattr(name))
             for name in self.dir()
         )
         return self.needs_type_completions(), tuples
@@ -570,4 +557,6 @@ def _is_class_instance(obj):
     except AttributeError:
         return False
     else:
-        return cls != type and not issubclass(cls, NOT_CLASS_TYPES)
+        # The isinstance check for cls is just there so issubclass doesn't
+        # raise an exception.
+        return cls != type and isinstance(cls, type) and not issubclass(cls, NOT_CLASS_TYPES)

jedi/inference/compiled/getattr_static.py

@@ -6,7 +6,7 @@ information returned to enable Jedi to make decisions.
 
 import types
 
-from jedi._compatibility import py_version
+from jedi import debug
 
 _sentinel = object()
 
@@ -38,7 +38,7 @@ def _is_type(obj):
     return True
 
 
-def _shadowed_dict_newstyle(klass):
+def _shadowed_dict(klass):
     dict_attr = type.__dict__["__dict__"]
     for entry in _static_getmro(klass):
         try:
@@ -53,74 +53,19 @@ def _shadowed_dict_newstyle(klass):
     return _sentinel
 
 
-def _static_getmro_newstyle(klass):
-    return type.__dict__['__mro__'].__get__(klass)
-
-
-if py_version >= 30:
-    _shadowed_dict = _shadowed_dict_newstyle
-    _get_type = type
-    _static_getmro = _static_getmro_newstyle
-else:
-    def _shadowed_dict(klass):
-        """
-        In Python 2 __dict__ is not overwritable:
-
-            class Foo(object): pass
-            setattr(Foo, '__dict__', 4)
-
-            Traceback (most recent call last):
-              File "<stdin>", line 1, in <module>
-            TypeError: __dict__ must be a dictionary object
-
-        It applies to both newstyle and oldstyle classes:
-
-            class Foo(object): pass
-            setattr(Foo, '__dict__', 4)
-            Traceback (most recent call last):
-              File "<stdin>", line 1, in <module>
-            AttributeError: attribute '__dict__' of 'type' objects is not writable
-
-        It also applies to instances of those objects. However to keep things
-        straight forward, newstyle classes always use the complicated way of
-        accessing it while oldstyle classes just use getattr.
-        """
-        if type(klass) is _oldstyle_class_type:
-            return getattr(klass, '__dict__', _sentinel)
-        return _shadowed_dict_newstyle(klass)
-
-    class _OldStyleClass:
-        pass
-
-    _oldstyle_instance_type = type(_OldStyleClass())
-    _oldstyle_class_type = type(_OldStyleClass)
-
-    def _get_type(obj):
-        type_ = object.__getattribute__(obj, '__class__')
-        if type_ is _oldstyle_instance_type:
-            # Somehow for old style classes we need to access it directly.
-            return obj.__class__
-        return type_
-
-    def _static_getmro(klass):
-        if type(klass) is _oldstyle_class_type:
-            def oldstyle_mro(klass):
-                """
-                Oldstyle mro is a really simplistic way of look up mro:
-                https://stackoverflow.com/questions/54867/what-is-the-difference-between-old-style-and-new-style-classes-in-python
-                """
-                yield klass
-                for base in klass.__bases__:
-                    for yield_from in oldstyle_mro(base):
-                        yield yield_from
-
-            return oldstyle_mro(klass)
-
-        return _static_getmro_newstyle(klass)
+def _static_getmro(klass):
+    mro = type.__dict__['__mro__'].__get__(klass)
+    if not isinstance(mro, (tuple, list)):
+        # There are unfortunately no tests for this, I was not able to
+        # reproduce this in pure Python. However should still solve the issue
+        # raised in GH #1517.
+        debug.warning('mro of %s returned %s, should be a tuple' % (klass, mro))
+        return ()
+    return mro
 
 
 def _safe_hasattr(obj, name):
-    return _check_class(_get_type(obj), name) is not _sentinel
+    return _check_class(type(obj), name) is not _sentinel
 
 
 def _safe_is_data_descriptor(obj):
@@ -143,7 +88,7 @@ def getattr_static(obj, attr, default=_sentinel):
     """
     instance_result = _sentinel
     if not _is_type(obj):
-        klass = _get_type(obj)
+        klass = type(obj)
         dict_attr = _shadowed_dict(klass)
         if (dict_attr is _sentinel or type(dict_attr) is types.MemberDescriptorType):
             instance_result = _check_instance(obj, attr)

jedi/inference/compiled/mixed.py

@@ -3,12 +3,10 @@ Used only for REPL Completion.
 """
 
 import inspect
-import os
-import sys
+from pathlib import Path
 
 from jedi.parser_utils import get_cached_code_lines
 
-from jedi._compatibility import unwrap
 from jedi import settings
 from jedi.cache import memoize_method
 from jedi.inference import compiled
@@ -36,7 +34,7 @@ class MixedObject(ValueWrapper):
 
     This combined logic makes it possible to provide more powerful REPL
     completion. It allows side effects that are not noticable with the default
-    parser structure to still be completeable.
+    parser structure to still be completable.
 
     The biggest difference from CompiledValue to MixedObject is that we are
     generally dealing with Python code and not with C code. This will generate
@@ -44,7 +42,7 @@ class MixedObject(ValueWrapper):
     to modify the runtime.
     """
     def __init__(self, compiled_value, tree_value):
-        super(MixedObject, self).__init__(tree_value)
+        super().__init__(tree_value)
         self.compiled_value = compiled_value
         self.access_handle = compiled_value.access_handle
 
@@ -71,12 +69,22 @@ class MixedObject(ValueWrapper):
         else:
             return self.compiled_value.get_safe_value(default)
 
+    @property
+    def array_type(self):
+        return self.compiled_value.array_type
+
+    def get_key_values(self):
+        return self.compiled_value.get_key_values()
+
     def py__simple_getitem__(self, index):
         python_object = self.compiled_value.access_handle.access._obj
         if type(python_object) in ALLOWED_GETITEM_TYPES:
             return self.compiled_value.py__simple_getitem__(index)
         return self._wrapped_value.py__simple_getitem__(index)
 
+    def negate(self):
+        return self.compiled_value.negate()
+
     def _as_context(self):
         if self.parent_context is None:
             return MixedModuleContext(self)
@@ -105,7 +113,7 @@ class MixedName(NameWrapper):
     The ``CompiledName._compiled_value`` is our MixedObject.
     """
     def __init__(self, wrapped_name, parent_tree_value):
-        super(MixedName, self).__init__(wrapped_name)
+        super().__init__(wrapped_name)
         self._parent_tree_value = parent_tree_value
 
     @property
@@ -131,12 +139,12 @@ class MixedName(NameWrapper):
 
 class MixedObjectFilter(compiled.CompiledValueFilter):
     def __init__(self, inference_state, compiled_value, tree_value):
-        super(MixedObjectFilter, self).__init__(inference_state, compiled_value)
+        super().__init__(inference_state, compiled_value)
         self._tree_value = tree_value
 
-    def _create_name(self, name):
+    def _create_name(self, *args, **kwargs):
         return MixedName(
-            super(MixedObjectFilter, self)._create_name(name),
+            super()._create_name(*args, **kwargs),
             self._tree_value,
         )
 
@@ -153,12 +161,11 @@ def _load_module(inference_state, path):
 
 def _get_object_to_check(python_object):
     """Check if inspect.getfile has a chance to find the source."""
-    if sys.version_info[0] > 2:
-        try:
-            python_object = unwrap(python_object)
-        except ValueError:
-            # Can return a ValueError when it wraps around
-            pass
+    try:
+        python_object = inspect.unwrap(python_object)
+    except ValueError:
+        # Can return a ValueError when it wraps around
+        pass
 
     if (inspect.ismodule(python_object)
             or inspect.isclass(python_object)
@@ -180,11 +187,19 @@ def _find_syntax_node_name(inference_state, python_object):
     try:
         python_object = _get_object_to_check(python_object)
         path = inspect.getsourcefile(python_object)
-    except TypeError:
+    except (OSError, TypeError):
         # The type might not be known (e.g. class_with_dict.__weakref__)
         return None
-    if path is None or not os.path.exists(path):
-        # The path might not exist or be e.g. <stdin>.
+    path = None if path is None else Path(path)
+    try:
+        if path is None or not path.exists():
+            # The path might not exist or be e.g. <stdin>.
+            return None
+    except OSError:
+        # Might raise an OSError on Windows:
+        #
+        #     [WinError 123] The filename, directory name, or volume label
+        #     syntax is incorrect: '<string>'
         return None
 
     file_io = FileIO(path)
@@ -252,7 +267,7 @@ def _find_syntax_node_name(inference_state, python_object):
 @inference_state_function_cache()
 def _create(inference_state, compiled_value, module_context):
     # TODO accessing this is bad, but it probably doesn't matter that much,
-    # because we're working with interpreteters only here.
+    # because we're working with interpreters only here.
     python_object = compiled_value.access_handle.access._obj
     result = _find_syntax_node_name(inference_state, python_object)
     if result is None:

jedi/inference/compiled/subprocess/__init__.py

@@ -5,23 +5,37 @@ goals:
 1. Making it safer - Segfaults and RuntimeErrors as well as stdout/stderr can
    be ignored and dealt with.
 2. Make it possible to handle different Python versions as well as virtualenvs.
+
+The architecture here is briefly:
+ - For each Jedi `Environment` there is a corresponding subprocess which
+   operates within the target environment. If the subprocess dies it is replaced
+   at this level.
+ - `CompiledSubprocess` manages exactly one subprocess and handles communication
+   from the parent side.
+ - `Listener` runs within the subprocess, processing each request and yielding
+   results.
+ - `InterpreterEnvironment` provides an API which matches that of `Environment`,
+   but runs functionality inline rather than within a subprocess. It is thus
+   used both directly in places where a subprocess is unnecessary and/or
+   undesirable and also within subprocesses themselves.
+ - `InferenceStateSubprocess` (or `InferenceStateSameProcess`) provide high
+   level access to functionality within the subprocess from within the parent.
+   Each `InterpreterState` has an instance of one of these, provided by its
+   environment.
 """
 
+import collections
 import os
 import sys
+import queue
 import subprocess
-import socket
-import errno
 import traceback
+import weakref
 from functools import partial
 from threading import Thread
-try:
-    from queue import Queue, Empty
-except ImportError:
-    from Queue import Queue, Empty  # python 2.7
+from typing import Dict, TYPE_CHECKING
 
-from jedi._compatibility import queue, is_py3, force_unicode, \
-    pickle_dump, pickle_load, GeneralizedPopen, weakref
+from jedi._compatibility import pickle_dump, pickle_load
 from jedi import debug
 from jedi.cache import memoize_method
 from jedi.inference.compiled.subprocess import functions
@@ -29,13 +43,32 @@ from jedi.inference.compiled.access import DirectObjectAccess, AccessPath, \
     SignatureParam
 from jedi.api.exceptions import InternalError
 
+if TYPE_CHECKING:
+    from jedi.inference import InferenceState
+
 
 _MAIN_PATH = os.path.join(os.path.dirname(__file__), '__main__.py')
+PICKLE_PROTOCOL = 4
 
 
-def _enqueue_output(out, queue):
+def _GeneralizedPopen(*args, **kwargs):
+    if os.name == 'nt':
+        try:
+            # Was introduced in Python 3.7.
+            CREATE_NO_WINDOW = subprocess.CREATE_NO_WINDOW
+        except AttributeError:
+            CREATE_NO_WINDOW = 0x08000000
+        kwargs['creationflags'] = CREATE_NO_WINDOW
+    # The child process doesn't need file descriptors except 0, 1, 2.
+    # This is unix only.
+    kwargs['close_fds'] = 'posix' in sys.builtin_module_names
+
+    return subprocess.Popen(*args, **kwargs)
+
+
+def _enqueue_output(out, queue_):
     for line in iter(out.readline, b''):
-        queue.put(line)
+        queue_.put(line)
 
 
 def _add_stderr_to_debug(stderr_queue):
@@ -46,7 +79,7 @@ def _add_stderr_to_debug(stderr_queue):
             line = stderr_queue.get_nowait()
             line = line.decode('utf-8', 'replace')
             debug.warning('stderr output: %s' % line.rstrip('\n'))
-        except Empty:
+        except queue.Empty:
             break
 
 
@@ -70,11 +103,10 @@ def _cleanup_process(process, thread):
             pass
 
 
-class _InferenceStateProcess(object):
-    def __init__(self, inference_state):
+class _InferenceStateProcess:
+    def __init__(self, inference_state: 'InferenceState') -> None:
         self._inference_state_weakref = weakref.ref(inference_state)
-        self._inference_state_id = id(inference_state)
-        self._handles = {}
+        self._handles: Dict[int, AccessHandle] = {}
 
     def get_or_create_access_handle(self, obj):
         id_ = id(obj)
@@ -104,11 +136,49 @@ class InferenceStateSameProcess(_InferenceStateProcess):
 
 
 class InferenceStateSubprocess(_InferenceStateProcess):
-    def __init__(self, inference_state, compiled_subprocess):
-        super(InferenceStateSubprocess, self).__init__(inference_state)
+    """
+    API to functionality which will run in a subprocess.
+
+    This mediates the interaction between an `InferenceState` and the actual
+    execution of functionality running within a `CompiledSubprocess`. Available
+    functions are defined in `.functions`, though should be accessed via
+    attributes on this class of the same name.
+
+    This class is responsible for indicating that the `InferenceState` within
+    the subprocess can be removed once the corresponding instance in the parent
+    goes away.
+    """
+
+    def __init__(
+        self,
+        inference_state: 'InferenceState',
+        compiled_subprocess: 'CompiledSubprocess',
+    ) -> None:
+        super().__init__(inference_state)
         self._used = False
         self._compiled_subprocess = compiled_subprocess
 
+        # Opaque id we'll pass to the subprocess to identify the context (an
+        # `InferenceState`) which should be used for the request. This allows us
+        # to make subsequent requests which operate on results from previous
+        # ones, while keeping a single subprocess which can work with several
+        # contexts in the parent process. Once it is no longer needed(i.e: when
+        # this class goes away), we also use this id to indicate that the
+        # subprocess can discard the context.
+        #
+        # Note: this id is deliberately coupled to this class (and not to
+        # `InferenceState`) as this class manages access handle mappings which
+        # must correspond to those in the subprocess. This approach also avoids
+        # race conditions from successive `InferenceState`s with the same object
+        # id (as observed while adding support for Python 3.13).
+        #
+        # This value does not need to be the `id()` of this instance, we merely
+        # need to ensure that it enables the (visible) lifetime of the context
+        # within the subprocess to match that of this class. We therefore also
+        # depend on the semantics of `CompiledSubprocess.delete_inference_state`
+        # for correctness.
+        self._inference_state_id = id(self)
+
     def __getattr__(self, name):
         func = _get_function(name)
 
@@ -116,7 +186,7 @@ class InferenceStateSubprocess(_InferenceStateProcess):
             self._used = True
 
             result = self._compiled_subprocess.run(
-                self._inference_state_weakref(),
+                self._inference_state_id,
                 func,
                 args=args,
                 kwargs=kwargs,
@@ -151,22 +221,31 @@ class InferenceStateSubprocess(_InferenceStateProcess):
             self._compiled_subprocess.delete_inference_state(self._inference_state_id)
 
 
-class CompiledSubprocess(object):
-    is_crashed = False
-    # Start with 2, gets set after _get_info.
-    _pickle_protocol = 2
+class CompiledSubprocess:
+    """
+    A subprocess which runs inference within a target environment.
 
-    def __init__(self, executable):
+    This class manages the interface to a single instance of such a process as
+    well as the lifecycle of the process itself. See `.__main__` and `Listener`
+    for the implementation of the subprocess and details of the protocol.
+
+    A single live instance of this is maintained by `jedi.api.environment.Environment`,
+    so that typically a single subprocess is used at a time.
+    """
+
+    is_crashed = False
+
+    def __init__(self, executable, env_vars=None):
         self._executable = executable
-        self._inference_state_deletion_queue = queue.deque()
+        self._env_vars = env_vars
+        self._inference_state_deletion_queue = collections.deque()
         self._cleanup_callable = lambda: None
 
     def __repr__(self):
         pid = os.getpid()
-        return '<%s _executable=%r, _pickle_protocol=%r, is_crashed=%r, pid=%r>' % (
+        return '<%s _executable=%r, is_crashed=%r, pid=%r>' % (
             self.__class__.__name__,
             self._executable,
-            self._pickle_protocol,
             self.is_crashed,
             pid,
         )
@@ -181,16 +260,14 @@ class CompiledSubprocess(object):
             os.path.dirname(os.path.dirname(parso_path)),
             '.'.join(str(x) for x in sys.version_info[:3]),
         )
-        process = GeneralizedPopen(
+        process = _GeneralizedPopen(
             args,
             stdin=subprocess.PIPE,
             stdout=subprocess.PIPE,
             stderr=subprocess.PIPE,
-            # Use system default buffering on Python 2 to improve performance
-            # (this is already the case on Python 3).
-            bufsize=-1
+            env=self._env_vars
         )
-        self._stderr_queue = Queue()
+        self._stderr_queue = queue.Queue()
         self._stderr_thread = t = Thread(
             target=_enqueue_output,
             args=(process.stderr, self._stderr_queue)
@@ -205,18 +282,18 @@ class CompiledSubprocess(object):
                                                   t)
         return process
 
-    def run(self, inference_state, function, args=(), kwargs={}):
+    def run(self, inference_state_id, function, args=(), kwargs={}):
         # Delete old inference_states.
         while True:
             try:
-                inference_state_id = self._inference_state_deletion_queue.pop()
+                delete_id = self._inference_state_deletion_queue.pop()
             except IndexError:
                 break
             else:
-                self._send(inference_state_id, None)
+                self._send(delete_id, None)
 
         assert callable(function)
-        return self._send(id(inference_state), function, args, kwargs)
+        return self._send(inference_state_id, function, args, kwargs)
 
     def get_sys_path(self):
         return self._send(None, functions.get_sys_path, (), {})
@@ -229,20 +306,10 @@ class CompiledSubprocess(object):
         if self.is_crashed:
             raise InternalError("The subprocess %s has crashed." % self._executable)
 
-        if not is_py3:
-            # Python 2 compatibility
-            kwargs = {force_unicode(key): value for key, value in kwargs.items()}
-
         data = inference_state_id, function, args, kwargs
         try:
-            pickle_dump(data, self._get_process().stdin, self._pickle_protocol)
-        except (socket.error, IOError) as e:
-            # Once Python2 will be removed we can just use `BrokenPipeError`.
-            # Also, somehow in windows it returns EINVAL instead of EPIPE if
-            # the subprocess dies.
-            if e.errno not in (errno.EPIPE, errno.EINVAL):
-                # Not a broken pipe
-                raise
+            pickle_dump(data, self._get_process().stdin, PICKLE_PROTOCOL)
+        except BrokenPipeError:
             self._kill()
             raise InternalError("The subprocess %s was killed. Maybe out of memory?"
                                 % self._executable)
@@ -274,22 +341,65 @@ class CompiledSubprocess(object):
 
     def delete_inference_state(self, inference_state_id):
         """
-        Currently we are not deleting inference_state instantly. They only get
-        deleted once the subprocess is used again. It would probably a better
-        solution to move all of this into a thread. However, the memory usage
-        of a single inference_state shouldn't be that high.
+        Indicate that an inference state (in the subprocess) is no longer
+        needed.
+
+        The state corresponding to the given id will become inaccessible and the
+        id may safely be re-used to refer to a different context.
+
+        Note: it is not guaranteed that the corresponding state will actually be
+        deleted immediately.
         """
-        # With an argument - the inference_state gets deleted.
+        # Warning: if changing the semantics of context deletion see the comment
+        # in `InferenceStateSubprocess.__init__` regarding potential race
+        # conditions.
+
+        # Currently we are not deleting the related state instantly. They only
+        # get deleted once the subprocess is used again. It would probably a
+        # better solution to move all of this into a thread. However, the memory
+        # usage of a single inference_state shouldn't be that high.
         self._inference_state_deletion_queue.append(inference_state_id)
 
 
-class Listener(object):
-    def __init__(self, pickle_protocol):
+class Listener:
+    """
+    Main loop for the subprocess which actually does the inference.
+
+    This class runs within the target environment. It listens to instructions
+    from the parent process, runs inference and returns the results.
+
+    The subprocess has a long lifetime and is expected to process several
+    requests, including for different `InferenceState` instances in the parent.
+    See `CompiledSubprocess` for the parent half of the system.
+
+    Communication is via pickled data sent serially over stdin and stdout.
+    Stderr is read only if the child process crashes.
+
+    The request protocol is a 4-tuple of:
+     * inference_state_id | None: an opaque identifier of the parent's
+       `InferenceState`. An `InferenceState` operating over an
+       `InterpreterEnvironment` is created within this process for each of
+       these, ensuring that each parent context has a corresponding context
+       here. This allows context to be persisted between requests. Unless
+       `None`, the local `InferenceState` will be passed to the given function
+       as the first positional argument.
+     * function | None: the function to run. This is expected to be a member of
+       `.functions`. `None` indicates that the corresponding inference state is
+       no longer needed and should be dropped.
+     * args: positional arguments to the `function`. If any of these are
+       `AccessHandle` instances they will be adapted to the local
+       `InferenceState` before being passed.
+     * kwargs: keyword arguments to the `function`. If any of these are
+       `AccessHandle` instances they will be adapted to the local
+       `InferenceState` before being passed.
+
+    The result protocol is a 3-tuple of either:
+     * (False, None, function result): if the function returns without error, or
+     * (True, traceback, exception): if the function raises an exception
+    """
+
+    def __init__(self):
         self._inference_states = {}
-        # TODO refactor so we don't need to process anymore just handle
-        # controlling.
-        self._process = _InferenceStateProcess(Listener)
-        self._pickle_protocol = pickle_protocol
 
     def _get_inference_state(self, function, inference_state_id):
         from jedi.inference import InferenceState
@@ -297,7 +407,7 @@ class Listener(object):
         try:
             inference_state = self._inference_states[inference_state_id]
         except KeyError:
-            from jedi.api.environment import InterpreterEnvironment
+            from jedi import InterpreterEnvironment
             inference_state = InferenceState(
                 # The project is not actually needed. Nothing should need to
                 # access it.
@@ -311,6 +421,9 @@ class Listener(object):
         if inference_state_id is None:
             return function(*args, **kwargs)
         elif function is None:
+            # Warning: if changing the semantics of context deletion see the comment
+            # in `InferenceStateSubprocess.__init__` regarding potential race
+            # conditions.
             del self._inference_states[inference_state_id]
         else:
             inference_state = self._get_inference_state(function, inference_state_id)
@@ -332,15 +445,8 @@ class Listener(object):
         # because stdout is used for IPC.
         sys.stdout = open(os.devnull, 'w')
         stdin = sys.stdin
-        if sys.version_info[0] > 2:
-            stdout = stdout.buffer
-            stdin = stdin.buffer
-        # Python 2 opens streams in text mode on Windows. Set stdout and stdin
-        # to binary mode.
-        elif sys.platform == 'win32':
-            import msvcrt
-            msvcrt.setmode(stdout.fileno(), os.O_BINARY)
-            msvcrt.setmode(stdin.fileno(), os.O_BINARY)
+        stdout = stdout.buffer
+        stdin = stdin.buffer
 
         while True:
             try:
@@ -354,11 +460,16 @@ class Listener(object):
             except Exception as e:
                 result = True, traceback.format_exc(), e
 
-            pickle_dump(result, stdout, self._pickle_protocol)
+            pickle_dump(result, stdout, PICKLE_PROTOCOL)
 
 
-class AccessHandle(object):
-    def __init__(self, subprocess, access, id_):
+class AccessHandle:
+    def __init__(
+        self,
+        subprocess: _InferenceStateProcess,
+        access: DirectObjectAccess,
+        id_: int,
+    ) -> None:
         self.access = access
         self._subprocess = subprocess
         self.id = id_
@@ -383,9 +494,8 @@ class AccessHandle(object):
         if name in ('id', 'access') or name.startswith('_'):
             raise AttributeError("Something went wrong with unpickling")
 
-        # if not is_py3: print >> sys.stderr, name
         # print('getattr', name, file=sys.stderr)
-        return partial(self._workaround, force_unicode(name))
+        return partial(self._workaround, name)
 
     def _workaround(self, name, *args, **kwargs):
         """

jedi/inference/compiled/subprocess/__main__.py

@@ -1,5 +1,11 @@
 import os
 import sys
+from importlib.abc import MetaPathFinder
+from importlib.machinery import PathFinder
+
+# Remove the first entry, because it's simply a directory entry that equals
+# this directory.
+del sys.path[0]
 
 
 def _get_paths():
@@ -11,45 +17,24 @@ def _get_paths():
     return {'jedi': _jedi_path, 'parso': _parso_path}
 
 
-# Remove the first entry, because it's simply a directory entry that equals
-# this directory.
-del sys.path[0]
+class _ExactImporter(MetaPathFinder):
+    def __init__(self, path_dct):
+        self._path_dct = path_dct
 
-if sys.version_info > (3, 4):
-    from importlib.machinery import PathFinder
+    def find_spec(self, fullname, path=None, target=None):
+        if path is None and fullname in self._path_dct:
+            p = self._path_dct[fullname]
+            spec = PathFinder.find_spec(fullname, path=[p], target=target)
+            return spec
+        return None
 
-    class _ExactImporter(object):
-        def __init__(self, path_dct):
-            self._path_dct = path_dct
-
-        def find_module(self, fullname, path=None):
-            if path is None and fullname in self._path_dct:
-                p = self._path_dct[fullname]
-                loader = PathFinder.find_module(fullname, path=[p])
-                return loader
-            return None
-
-    # Try to import jedi/parso.
-    sys.meta_path.insert(0, _ExactImporter(_get_paths()))
-    from jedi.inference.compiled import subprocess  # NOQA
-    sys.meta_path.pop(0)
-else:
-    import imp
-
-    def load(name):
-        paths = list(_get_paths().values())
-        fp, pathname, description = imp.find_module(name, paths)
-        return imp.load_module(name, fp, pathname, description)
-
-    load('parso')
-    load('jedi')
-    from jedi.inference.compiled import subprocess  # NOQA
-
-from jedi._compatibility import highest_pickle_protocol  # noqa: E402
 
+# Try to import jedi/parso.
+sys.meta_path.insert(0, _ExactImporter(_get_paths()))
+from jedi.inference.compiled import subprocess  # noqa: E402
+sys.meta_path.pop(0)
 
 # Retrieve the pickle protocol.
 host_sys_version = [int(x) for x in sys.argv[2].split('.')]
-pickle_protocol = highest_pickle_protocol([sys.version_info, host_sys_version])
 # And finally start the client.
-subprocess.Listener(pickle_protocol=pickle_protocol).listen()
+subprocess.Listener().listen()

jedi/inference/compiled/subprocess/functions.py

@@ -1,15 +1,20 @@
-from __future__ import print_function
 import sys
 import os
+import inspect
+import importlib
+from pathlib import Path
+from zipfile import ZipFile
+from zipimport import zipimporter, ZipImportError
+from importlib.machinery import all_suffixes
 
-from jedi._compatibility import find_module, cast_path, force_unicode, \
-    iter_modules, all_suffixes
 from jedi.inference.compiled import access
+from jedi import debug
 from jedi import parser_utils
+from jedi.file_io import KnownContentFileIO, ZipFileIO
 
 
 def get_sys_path():
-    return list(map(cast_path, sys.path))
+    return sys.path
 
 
 def load_module(inference_state, **kwargs):
@@ -32,7 +37,7 @@ def get_module_info(inference_state, sys_path=None, full_name=None, **kwargs):
     if sys_path is not None:
         sys.path, temp = sys_path, sys.path
     try:
-        return find_module(full_name=full_name, **kwargs)
+        return _find_module(full_name=full_name, **kwargs)
     except ImportError:
         return None, None
     finally:
@@ -40,15 +45,8 @@ def get_module_info(inference_state, sys_path=None, full_name=None, **kwargs):
             sys.path = temp
 
 
-def list_module_names(inference_state, search_path):
-    return [
-        force_unicode(name)
-        for module_loader, name, is_pkg in iter_modules(search_path)
-    ]
-
-
 def get_builtin_module_names(inference_state):
-    return list(map(force_unicode, sys.builtin_module_names))
+    return sys.builtin_module_names
 
 
 def _test_raise_error(inference_state, exception_type):
@@ -84,3 +82,176 @@ def _get_init_path(directory_path):
 
 def safe_literal_eval(inference_state, value):
     return parser_utils.safe_literal_eval(value)
+
+
+def iter_module_names(*args, **kwargs):
+    return list(_iter_module_names(*args, **kwargs))
+
+
+def _iter_module_names(inference_state, paths):
+    # Python modules/packages
+    for path in paths:
+        try:
+            dir_entries = ((entry.name, entry.is_dir()) for entry in os.scandir(path))
+        except OSError:
+            try:
+                zip_import_info = zipimporter(path)
+                # Unfortunately, there is no public way to access zipimporter's
+                # private _files member. We therefore have to use a
+                # custom function to iterate over the files.
+                dir_entries = _zip_list_subdirectory(
+                    zip_import_info.archive, zip_import_info.prefix)
+            except ZipImportError:
+                # The file might not exist or reading it might lead to an error.
+                debug.warning("Not possible to list directory: %s", path)
+                continue
+        for name, is_dir in dir_entries:
+            # First Namespaces then modules/stubs
+            if is_dir:
+                # pycache is obviously not an interesting namespace. Also the
+                # name must be a valid identifier.
+                if name != '__pycache__' and name.isidentifier():
+                    yield name
+            else:
+                if name.endswith('.pyi'):  # Stub files
+                    modname = name[:-4]
+                else:
+                    modname = inspect.getmodulename(name)
+
+                if modname and '.' not in modname:
+                    if modname != '__init__':
+                        yield modname
+
+
+def _find_module(string, path=None, full_name=None, is_global_search=True):
+    """
+    Provides information about a module.
+
+    This function isolates the differences in importing libraries introduced with
+    python 3.3 on; it gets a module name and optionally a path. It will return a
+    tuple containin an open file for the module (if not builtin), the filename
+    or the name of the module if it is a builtin one and a boolean indicating
+    if the module is contained in a package.
+    """
+    spec = None
+    loader = None
+
+    for finder in sys.meta_path:
+        if is_global_search and finder != importlib.machinery.PathFinder:
+            p = None
+        else:
+            p = path
+        try:
+            find_spec = finder.find_spec
+        except AttributeError:
+            # These are old-school clases that still have a different API, just
+            # ignore those.
+            continue
+
+        spec = find_spec(string, p)
+        if spec is not None:
+            if spec.origin == "frozen":
+                continue
+
+            loader = spec.loader
+
+            if loader is None and not spec.has_location:
+                # This is a namespace package.
+                full_name = string if not path else full_name
+                implicit_ns_info = ImplicitNSInfo(full_name, spec.submodule_search_locations._path)
+                return implicit_ns_info, True
+            break
+
+    return _find_module_py33(string, path, loader)
+
+
+def _find_module_py33(string, path=None, loader=None, full_name=None, is_global_search=True):
+    if not loader:
+        spec = importlib.machinery.PathFinder.find_spec(string, path)
+        if spec is not None:
+            loader = spec.loader
+
+    if loader is None and path is None:  # Fallback to find builtins
+        try:
+            spec = importlib.util.find_spec(string)
+            if spec is not None:
+                loader = spec.loader
+        except ValueError as e:
+            # See #491. Importlib might raise a ValueError, to avoid this, we
+            # just raise an ImportError to fix the issue.
+            raise ImportError("Originally  " + repr(e))
+
+    if loader is None:
+        raise ImportError("Couldn't find a loader for {}".format(string))
+
+    return _from_loader(loader, string)
+
+
+def _from_loader(loader, string):
+    try:
+        is_package_method = loader.is_package
+    except AttributeError:
+        is_package = False
+    else:
+        is_package = is_package_method(string)
+    try:
+        get_filename = loader.get_filename
+    except AttributeError:
+        return None, is_package
+    else:
+        module_path = get_filename(string)
+
+    # To avoid unicode and read bytes, "overwrite" loader.get_source if
+    # possible.
+    try:
+        f = type(loader).get_source
+    except AttributeError:
+        raise ImportError("get_source was not defined on loader")
+
+    if f is not importlib.machinery.SourceFileLoader.get_source:
+        # Unfortunately we are reading unicode here, not bytes.
+        # It seems hard to get bytes, because the zip importer
+        # logic just unpacks the zip file and returns a file descriptor
+        # that we cannot as easily access. Therefore we just read it as
+        # a string in the cases where get_source was overwritten.
+        code = loader.get_source(string)
+    else:
+        code = _get_source(loader, string)
+
+    if code is None:
+        return None, is_package
+    if isinstance(loader, zipimporter):
+        return ZipFileIO(module_path, code, Path(loader.archive)), is_package
+
+    return KnownContentFileIO(module_path, code), is_package
+
+
+def _get_source(loader, fullname):
+    """
+    This method is here as a replacement for SourceLoader.get_source. That
+    method returns unicode, but we prefer bytes.
+    """
+    path = loader.get_filename(fullname)
+    try:
+        return loader.get_data(path)
+    except OSError:
+        raise ImportError('source not available through get_data()',
+                          name=fullname)
+
+
+def _zip_list_subdirectory(zip_path, zip_subdir_path):
+    zip_file = ZipFile(zip_path)
+    zip_subdir_path = Path(zip_subdir_path)
+    zip_content_file_paths = zip_file.namelist()
+    for raw_file_name in zip_content_file_paths:
+        file_path = Path(raw_file_name)
+        if file_path.parent == zip_subdir_path:
+            file_path = file_path.relative_to(zip_subdir_path)
+            yield file_path.name, raw_file_name.endswith("/")
+
+
+class ImplicitNSInfo:
+    """Stores information returned from an implicit namespace spec"""
+    def __init__(self, name, paths):
+        self.name = name
+        self.paths = paths

jedi/inference/compiled/value.py

@@ -3,10 +3,12 @@ Imitate the parser representation.
 """
 import re
 from functools import partial
+from inspect import Parameter
+from pathlib import Path
+from typing import Optional
 
 from jedi import debug
 from jedi.inference.utils import to_list
-from jedi._compatibility import force_unicode, Parameter, cast_path
 from jedi.cache import memoize_method
 from jedi.inference.filters import AbstractFilter
 from jedi.inference.names import AbstractNameDefinition, ValueNameMixin, \
@@ -20,8 +22,8 @@ from jedi.inference.signature import BuiltinSignature
 from jedi.inference.context import CompiledContext, CompiledModuleContext
 
 
-class CheckAttribute(object):
-    """Raises an AttributeError if the attribute X isn't available."""
+class CheckAttribute:
+    """Raises :exc:`AttributeError` if the attribute X is not available."""
     def __init__(self, check_name=None):
         # Remove the py in front of e.g. py__call__.
         self.check_name = check_name
@@ -29,7 +31,7 @@ class CheckAttribute(object):
     def __call__(self, func):
         self.func = func
         if self.check_name is None:
-            self.check_name = force_unicode(func.__name__[2:])
+            self.check_name = func.__name__[2:]
         return self
 
     def __get__(self, instance, owner):
@@ -43,22 +45,21 @@ class CheckAttribute(object):
 
 class CompiledValue(Value):
     def __init__(self, inference_state, access_handle, parent_context=None):
-        super(CompiledValue, self).__init__(inference_state, parent_context)
+        super().__init__(inference_state, parent_context)
         self.access_handle = access_handle
 
     def py__call__(self, arguments):
         return_annotation = self.access_handle.get_return_annotation()
         if return_annotation is not None:
-            # TODO the return annotation may also be a string.
             return create_from_access_path(
                 self.inference_state,
                 return_annotation
             ).execute_annotation()
 
         try:
-            self.access_handle.getattr_paths(u'__call__')
+            self.access_handle.getattr_paths('__call__')
         except AttributeError:
-            return super(CompiledValue, self).py__call__(arguments)
+            return super().py__call__(arguments)
         else:
             if self.access_handle.is_class():
                 from jedi.inference.value import CompiledInstance
@@ -161,11 +162,14 @@ class CompiledValue(Value):
     def py__simple_getitem__(self, index):
         with reraise_getitem_errors(IndexError, KeyError, TypeError):
             try:
-                access = self.access_handle.py__simple_getitem__(index)
+                access = self.access_handle.py__simple_getitem__(
+                    index,
+                    safe=not self.inference_state.allow_unsafe_executions
+                )
             except AttributeError:
-                return super(CompiledValue, self).py__simple_getitem__(index)
+                return super().py__simple_getitem__(index)
         if access is None:
-            return NO_VALUES
+            return super().py__simple_getitem__(index)
 
         return ValueSet([create_from_access_path(self.inference_state, access)])
 
@@ -174,20 +178,15 @@ class CompiledValue(Value):
         if all_access_paths is None:
             # This means basically that no __getitem__ has been defined on this
             # object.
-            return super(CompiledValue, self).py__getitem__(index_value_set, contextualized_node)
+            return super().py__getitem__(index_value_set, contextualized_node)
         return ValueSet(
             create_from_access_path(self.inference_state, access)
             for access in all_access_paths
         )
 
     def py__iter__(self, contextualized_node=None):
-        # Python iterators are a bit strange, because there's no need for
-        # the __iter__ function as long as __getitem__ is defined (it will
-        # just start with __getitem__(0). This is especially true for
-        # Python 2 strings, where `str.__iter__` is not even defined.
         if not self.access_handle.has_iter():
-            for x in super(CompiledValue, self).py__iter__(contextualized_node):
-                yield x
+            yield from super().py__iter__(contextualized_node)
 
         access_path_list = self.access_handle.py__iter__list()
         if access_path_list is None:
@@ -222,10 +221,8 @@ class CompiledValue(Value):
                 continue
             else:
                 bltn_obj = builtin_from_name(self.inference_state, name)
-                for result in self.inference_state.execute(bltn_obj, params):
-                    yield result
-        for type_ in docstrings.infer_return_types(self):
-            yield type_
+                yield from self.inference_state.execute(bltn_obj, params)
+        yield from docstrings.infer_return_types(self)
 
     def get_safe_value(self, default=_sentinel):
         try:
@@ -264,7 +261,7 @@ class CompiledValue(Value):
                 v.with_generics(arguments)
                 for v in self.inference_state.typing_module.py__getattribute__(name)
             ]).execute_annotation()
-        return super(CompiledValue, self).execute_annotation()
+        return super().execute_annotation()
 
     def negate(self):
         return create_from_access_path(self.inference_state, self.access_handle.negate())
@@ -285,6 +282,11 @@ class CompiledValue(Value):
             for k in self.access_handle.get_key_paths()
         ]
 
+    def get_type_hint(self, add_class_info=True):
+        if self.access_handle.get_repr() in ('None', "<class 'NoneType'>"):
+            return 'None'
+        return None
+
 
 class CompiledModule(CompiledValue):
     file_io = None  # For modules
@@ -293,10 +295,7 @@ class CompiledModule(CompiledValue):
         return CompiledModuleContext(self)
 
     def py__path__(self):
-        paths = self.access_handle.py__path__()
-        if paths is None:
-            return None
-        return map(cast_path, paths)
+        return self.access_handle.py__path__()
 
     def is_package(self):
         return self.py__path__() is not None
@@ -309,20 +308,20 @@ class CompiledModule(CompiledValue):
             return ()
         return tuple(name.split('.'))
 
-    def py__file__(self):
-        return cast_path(self.access_handle.py__file__())
+    def py__file__(self) -> Optional[Path]:
+        return self.access_handle.py__file__()  # type: ignore[no-any-return]
 
 
 class CompiledName(AbstractNameDefinition):
-    def __init__(self, inference_state, parent_value, name):
+    def __init__(self, inference_state, parent_value, name, is_descriptor):
         self._inference_state = inference_state
         self.parent_context = parent_value.as_context()
         self._parent_value = parent_value
         self.string_name = name
+        self.is_descriptor = is_descriptor
 
     def py__doc__(self):
-        value, = self.infer()
-        return value.py__doc__()
+        return self.infer_compiled_value().py__doc__()
 
     def _get_qualified_names(self):
         parent_qualified_names = self.parent_context.get_qualified_names()
@@ -346,16 +345,17 @@ class CompiledName(AbstractNameDefinition):
 
     @property
     def api_type(self):
-        api = self.infer()
-        # If we can't find the type, assume it is an instance variable
-        if not api:
+        if self.is_descriptor:
+            # In case of properties we want to avoid executions as much as
+            # possible. Since the api_type can be wrong for other reasons
+            # anyway, we just return instance here.
             return "instance"
-        return next(iter(api)).api_type
+        return self.infer_compiled_value().api_type
 
-    @memoize_method
     def infer(self):
         return ValueSet([self.infer_compiled_value()])
 
+    @memoize_method
     def infer_compiled_value(self):
         return create_from_name(self._inference_state, self._parent_value, self.string_name)
 
@@ -440,9 +440,10 @@ class CompiledValueFilter(AbstractFilter):
 
     def get(self, name):
         access_handle = self.compiled_value.access_handle
+        safe = not self._inference_state.allow_unsafe_executions
         return self._get(
             name,
-            lambda name, unsafe: access_handle.is_allowed_getattr(name, unsafe),
+            lambda name: access_handle.is_allowed_getattr(name, safe=safe),
             lambda name: name in access_handle.dir(),
             check_has_attribute=True
         )
@@ -451,39 +452,40 @@ class CompiledValueFilter(AbstractFilter):
         """
         To remove quite a few access calls we introduced the callback here.
         """
-        # Always use unicode objects in Python 2 from here.
-        name = force_unicode(name)
-
-        if self._inference_state.allow_descriptor_getattr:
-            pass
-
-        has_attribute, is_descriptor = allowed_getattr_callback(
+        has_attribute, is_descriptor, property_return_annotation = allowed_getattr_callback(
             name,
-            unsafe=self._inference_state.allow_descriptor_getattr
         )
+        if property_return_annotation is not None:
+            values = create_from_access_path(
+                self._inference_state,
+                property_return_annotation
+            ).execute_annotation()
+            if values:
+                return [CompiledValueName(v, name) for v in values]
+
         if check_has_attribute and not has_attribute:
             return []
 
         if (is_descriptor or not has_attribute) \
-                and not self._inference_state.allow_descriptor_getattr:
+                and not self._inference_state.allow_unsafe_executions:
             return [self._get_cached_name(name, is_empty=True)]
 
         if self.is_instance and not in_dir_callback(name):
             return []
-        return [self._get_cached_name(name)]
+        return [self._get_cached_name(name, is_descriptor=is_descriptor)]
 
     @memoize_method
-    def _get_cached_name(self, name, is_empty=False):
+    def _get_cached_name(self, name, is_empty=False, *, is_descriptor=False):
         if is_empty:
             return EmptyCompiledName(self._inference_state, name)
         else:
-            return self._create_name(name)
+            return self._create_name(name, is_descriptor=is_descriptor)
 
     def values(self):
         from jedi.inference.compiled import builtin_from_name
         names = []
         needs_type_completions, dir_infos = self.compiled_value.access_handle.get_dir_infos()
-        # We could use `unsafe` here as well, especially as a parameter to
+        # We could use `safe=False` here as well, especially as a parameter to
         # get_dir_infos. But this would lead to a lot of property executions
         # that are probably not wanted. The drawback for this is that we
         # have a different name for `get` and `values`. For `get` we always
@@ -491,21 +493,22 @@ class CompiledValueFilter(AbstractFilter):
         for name in dir_infos:
             names += self._get(
                 name,
-                lambda name, unsafe: dir_infos[name],
+                lambda name: dir_infos[name],
                 lambda name: name in dir_infos,
             )
 
         # ``dir`` doesn't include the type names.
         if not self.is_instance and needs_type_completions:
-            for filter in builtin_from_name(self._inference_state, u'type').get_filters():
+            for filter in builtin_from_name(self._inference_state, 'type').get_filters():
                 names += filter.values()
         return names
 
-    def _create_name(self, name):
+    def _create_name(self, name, is_descriptor):
         return CompiledName(
             self._inference_state,
             self.compiled_value,
-            name
+            name,
+            is_descriptor,
         )
 
     def __repr__(self):
@@ -513,11 +516,11 @@ class CompiledValueFilter(AbstractFilter):
 
 
 docstr_defaults = {
-    'floating point number': u'float',
-    'character': u'str',
-    'integer': u'int',
-    'dictionary': u'dict',
-    'string': u'str',
+    'floating point number': 'float',
+    'character': 'str',
+    'integer': 'int',
+    'dictionary': 'dict',
+    'string': 'str',
 }
 
 
@@ -529,7 +532,6 @@ def _parse_function_doc(doc):
     TODO docstrings like utime(path, (atime, mtime)) and a(b [, b]) -> None
     TODO docstrings like 'tuple of integers'
     """
-    doc = force_unicode(doc)
     # parse round parentheses: def func(a, (b,c))
     try:
         count = 0
@@ -548,7 +550,7 @@ def _parse_function_doc(doc):
         # UnboundLocalError for undefined end in last line
         debug.dbg('no brackets found - no param')
         end = 0
-        param_str = u''
+        param_str = ''
     else:
         # remove square brackets, that show an optional param ( = None)
         def change_options(m):
@@ -566,9 +568,9 @@ def _parse_function_doc(doc):
     param_str = param_str.replace('-', '_')  # see: isinstance.__doc__
 
     # parse return value
-    r = re.search(u'-[>-]* ', doc[end:end + 7])
+    r = re.search('-[>-]* ', doc[end:end + 7])
     if r is None:
-        ret = u''
+        ret = ''
     else:
         index = end + r.end()
         # get result type, which can contain newlines
@@ -584,9 +586,6 @@ def _parse_function_doc(doc):
 
 def create_from_name(inference_state, compiled_value, name):
     access_paths = compiled_value.access_handle.getattr_paths(name, default=None)
-    parent_context = compiled_value
-    if parent_context.is_class():
-        parent_context = parent_context.parent_context
 
     value = None
     for access_path in access_paths:

jedi/inference/context.py

@@ -1,5 +1,7 @@
 from abc import abstractmethod
 from contextlib import contextmanager
+from pathlib import Path
+from typing import Optional
 
 from parso.tree import search_ancestor
 from parso.python.tree import Name
@@ -13,7 +15,7 @@ from jedi import debug
 from jedi import parser_utils
 
 
-class AbstractContext(object):
+class AbstractContext:
     # Must be defined: inference_state and tree_node and parent_context as an attribute/property
 
     def __init__(self, inference_state):
@@ -129,6 +131,9 @@ class AbstractContext(object):
     def is_compiled(self):
         return False
 
+    def is_bound_method(self):
+        return False
+
     @abstractmethod
     def py__name__(self):
         raise NotImplementedError
@@ -161,7 +166,7 @@ class ValueContext(AbstractContext):
     Should be defined, otherwise the API returns empty types.
     """
     def __init__(self, value):
-        super(ValueContext, self).__init__(value.inference_state)
+        super().__init__(value.inference_state)
         self._value = value
 
     @property
@@ -190,6 +195,9 @@ class ValueContext(AbstractContext):
     def is_compiled(self):
         return self._value.is_compiled()
 
+    def is_bound_method(self):
+        return self._value.is_bound_method()
+
     def py__name__(self):
         return self._value.py__name__()
 
@@ -210,7 +218,7 @@ class ValueContext(AbstractContext):
         return '%s(%s)' % (self.__class__.__name__, self._value)
 
 
-class TreeContextMixin(object):
+class TreeContextMixin:
     def infer_node(self, node):
         from jedi.inference.syntax_tree import infer_node
         return infer_node(self, node)
@@ -249,8 +257,7 @@ class TreeContextMixin(object):
             if scope_node.type in ('funcdef', 'lambdef', 'classdef'):
                 return self.create_value(scope_node).as_context()
             elif scope_node.type in ('comp_for', 'sync_comp_for'):
-                parent_scope = parser_utils.get_parent_scope(scope_node)
-                parent_context = from_scope_node(parent_scope)
+                parent_context = from_scope_node(parent_scope(scope_node.parent))
                 if node.start_pos >= scope_node.children[-1].start_pos:
                     return parent_context
                 return CompForContext(parent_context, scope_node)
@@ -302,13 +309,13 @@ class FunctionContext(TreeContextMixin, ValueContext):
 
 
 class ModuleContext(TreeContextMixin, ValueContext):
-    def py__file__(self):
-        return self._value.py__file__()
+    def py__file__(self) -> Optional[Path]:
+        return self._value.py__file__()  # type: ignore[no-any-return]
 
     def get_filters(self, until_position=None, origin_scope=None):
         filters = self._value.get_filters(origin_scope)
         # Skip the first filter and replace it.
-        next(filters)
+        next(filters, None)
         yield MergedFilter(
             ParserTreeFilter(
                 parent_context=self,
@@ -317,11 +324,10 @@ class ModuleContext(TreeContextMixin, ValueContext):
             ),
             self.get_global_filter(),
         )
-        for f in filters:  # Python 2...
-            yield f
+        yield from filters
 
     def get_global_filter(self):
-        return GlobalNameFilter(self, self.tree_node)
+        return GlobalNameFilter(self)
 
     @property
     def string_names(self):
@@ -347,8 +353,12 @@ class NamespaceContext(TreeContextMixin, ValueContext):
     def get_value(self):
         return self._value
 
-    def py__file__(self):
-        return self._value.py__file__()
+    @property
+    def string_names(self):
+        return self._value.string_names
+
+    def py__file__(self) -> Optional[Path]:
+        return self._value.py__file__()  # type: ignore[no-any-return]
 
 
 class ClassContext(TreeContextMixin, ValueContext):
@@ -365,7 +375,7 @@ class ClassContext(TreeContextMixin, ValueContext):
 
 class CompForContext(TreeContextMixin, AbstractContext):
     def __init__(self, parent_context, comp_for):
-        super(CompForContext, self).__init__(parent_context.inference_state)
+        super().__init__(parent_context.inference_state)
         self.tree_node = comp_for
         self.parent_context = parent_context
 
@@ -397,8 +407,8 @@ class CompiledModuleContext(CompiledContext):
     def string_names(self):
         return self._value.string_names
 
-    def py__file__(self):
-        return self._value.py__file__()
+    def py__file__(self) -> Optional[Path]:
+        return self._value.py__file__()  # type: ignore[no-any-return]
 
 
 def _get_global_filters_for_name(context, name_or_none, position):
@@ -429,13 +439,12 @@ def get_global_filters(context, until_position, origin_scope):
     For global name lookups. The filters will handle name resolution
     themselves, but here we gather possible filters downwards.
 
-    >>> from jedi._compatibility import u, no_unicode_pprint
     >>> from jedi import Script
-    >>> script = Script(u('''
+    >>> script = Script('''
     ... x = ['a', 'b', 'c']
     ... def func():
     ...     y = None
-    ... '''))
+    ... ''')
     >>> module_node = script._module_node
     >>> scope = next(module_node.iter_funcdefs())
     >>> scope
@@ -445,7 +454,7 @@ def get_global_filters(context, until_position, origin_scope):
 
     First we get the names from the function scope.
 
-    >>> no_unicode_pprint(filters[0])  # doctest: +ELLIPSIS
+    >>> print(filters[0])  # doctest: +ELLIPSIS
     MergedFilter(<ParserTreeFilter: ...>, <GlobalNameFilter: ...>)
     >>> sorted(str(n) for n in filters[0].values())  # doctest: +NORMALIZE_WHITESPACE
     ['<TreeNameDefinition: string_name=func start_pos=(3, 4)>',
@@ -474,15 +483,17 @@ def get_global_filters(context, until_position, origin_scope):
     from jedi.inference.value.function import BaseFunctionExecutionContext
     while context is not None:
         # Names in methods cannot be resolved within the class.
-        for filter in context.get_filters(
-                until_position=until_position,
-                origin_scope=origin_scope):
-            yield filter
+        yield from context.get_filters(
+            until_position=until_position,
+            origin_scope=origin_scope
+        )
         if isinstance(context, (BaseFunctionExecutionContext, ModuleContext)):
             # The position should be reset if the current scope is a function.
             until_position = None
 
         context = context.parent_context
 
+    b = next(base_context.inference_state.builtins_module.get_filters(), None)
+    assert b is not None
     # Add builtins to the global scope.
-    yield next(base_context.inference_state.builtins_module.get_filters())
+    yield b

jedi/inference/docstring_utils.py

@@ -0,0 +1,21 @@
+from jedi.inference.value import ModuleValue
+from jedi.inference.context import ModuleContext
+
+
+class DocstringModule(ModuleValue):
+    def __init__(self, in_module_context, **kwargs):
+        super().__init__(**kwargs)
+        self._in_module_context = in_module_context
+
+    def _as_context(self):
+        return DocstringModuleContext(self, self._in_module_context)
+
+
+class DocstringModuleContext(ModuleContext):
+    def __init__(self, module_value, in_module_context):
+        super().__init__(module_value)
+        self._in_module_context = in_module_context
+
+    def get_filters(self, origin_scope=None, until_position=None):
+        yield from super().get_filters(until_position=until_position)
+        yield from self._in_module_context.get_filters()

jedi/inference/docstrings.py

@@ -1,7 +1,7 @@
 """
 Docstrings are another source of information for functions and classes.
-:mod:`jedi.inference.dynamic` tries to find all executions of functions, while
-the docstring parsing is much easier. There are three different types of
+:mod:`jedi.inference.dynamic_params` tries to find all executions of functions,
+while the docstring parsing is much easier. There are three different types of
 docstrings that |jedi| understands:
 
 - `Sphinx <http://sphinx-doc.org/markup/desc.html#info-field-lists>`_
@@ -17,13 +17,10 @@ annotations.
 
 import re
 import warnings
-from textwrap import dedent
 
 from parso import parse, ParserSyntaxError
 
-from jedi._compatibility import u
 from jedi import debug
-from jedi.inference.utils import indent_block
 from jedi.inference.cache import inference_state_method_cache
 from jedi.inference.base_value import iterator_to_value_set, ValueSet, \
     NO_VALUES
@@ -51,7 +48,7 @@ def _get_numpy_doc_string_cls():
     global _numpy_doc_string_cache
     if isinstance(_numpy_doc_string_cache, (ImportError, SyntaxError)):
         raise _numpy_doc_string_cache
-    from numpydoc.docscrape import NumpyDocString
+    from numpydoc.docscrape import NumpyDocString  # type: ignore[import]
     _numpy_doc_string_cache = NumpyDocString
     return _numpy_doc_string_cache
 
@@ -96,8 +93,7 @@ def _search_return_in_numpydocstr(docstr):
         # Return names are optional and if so the type is in the name
         if not r_type:
             r_type = r_name
-        for type_ in _expand_typestr(r_type):
-            yield type_
+        yield from _expand_typestr(r_type)
 
 
 def _expand_typestr(type_str):
@@ -115,7 +111,7 @@ def _expand_typestr(type_str):
     elif type_str.startswith('{'):
         node = parse(type_str, version='3.7').children[0]
         if node.type == 'atom':
-            for leaf in node.children[1].children:
+            for leaf in getattr(node.children[1], "children", []):
                 if leaf.type == 'number':
                     if '.' in leaf.value:
                         yield 'float'
@@ -184,55 +180,40 @@ def _strip_rst_role(type_str):
 
 
 def _infer_for_statement_string(module_context, string):
-    code = dedent(u("""
-    def pseudo_docstring_stuff():
-        '''
-        Create a pseudo function for docstring statements.
-        Need this docstring so that if the below part is not valid Python this
-        is still a function.
-        '''
-    {}
-    """))
     if string is None:
         return []
 
-    for element in re.findall(r'((?:\w+\.)*\w+)\.', string):
-        # Try to import module part in dotted name.
-        # (e.g., 'threading' in 'threading.Thread').
-        string = 'import %s\n' % element + string
+    potential_imports = re.findall(r'((?:\w+\.)*\w+)\.', string)
+    # Try to import module part in dotted name.
+    # (e.g., 'threading' in 'threading.Thread').
+    imports = "\n".join(f"import {p}" for p in potential_imports)
+    string = f'{imports}\n{string}'
 
-    # Take the default grammar here, if we load the Python 2.7 grammar here, it
-    # will be impossible to use `...` (Ellipsis) as a token. Docstring types
-    # don't need to conform with the current grammar.
     debug.dbg('Parse docstring code %s', string, color='BLUE')
-    grammar = module_context.inference_state.latest_grammar
+    grammar = module_context.inference_state.grammar
     try:
-        module = grammar.parse(code.format(indent_block(string)), error_recovery=False)
+        module = grammar.parse(string, error_recovery=False)
     except ParserSyntaxError:
         return []
     try:
-        funcdef = next(module.iter_funcdefs())
-        # First pick suite, then simple_stmt and then the node,
-        # which is also not the last item, because there's a newline.
-        stmt = funcdef.children[-1].children[-1].children[-2]
+        # It's not the last item, because that's an end marker.
+        stmt = module.children[-2]
     except (AttributeError, IndexError):
         return []
 
     if stmt.type not in ('name', 'atom', 'atom_expr'):
         return []
 
-    from jedi.inference.value import FunctionValue
-    function_value = FunctionValue(
-        module_context.inference_state,
-        module_context,
-        funcdef
+    # Here we basically use a fake module that also uses the filters in
+    # the actual module.
+    from jedi.inference.docstring_utils import DocstringModule
+    m = DocstringModule(
+        in_module_context=module_context,
+        inference_state=module_context.inference_state,
+        module_node=module,
+        code_lines=[],
     )
-    func_execution_context = function_value.as_context()
-    # Use the module of the param.
-    # TODO this module is not the module of the param in case of a function
-    # call. In that case it's the module of the function call.
-    # stuffed with content from a function call.
-    return list(_execute_types_in_stmt(func_execution_context, stmt))
+    return list(_execute_types_in_stmt(m.as_context(), stmt))
 
 
 def _execute_types_in_stmt(module_context, stmt):
@@ -299,9 +280,7 @@ def infer_return_types(function_value):
             if match:
                 yield _strip_rst_role(match.group(1))
         # Check for numpy style return hint
-        for type_ in _search_return_in_numpydocstr(code):
-            yield type_
+        yield from _search_return_in_numpydocstr(code)
 
     for type_str in search_return_in_docstr(function_value.py__doc__()):
-        for value in _infer_for_statement_string(function_value.get_root_context(), type_str):
-            yield value
+        yield from _infer_for_statement_string(function_value.get_root_context(), type_str)

jedi/inference/dynamic_params.py

@@ -48,7 +48,6 @@ def _avoid_recursions(func):
                 finally:
                     inf.dynamic_params_depth -= 1
             return NO_VALUES
-        return
     return wrapper
 
 
@@ -67,11 +66,11 @@ def dynamic_param_lookup(function_value, param_index):
     have to look for all calls to ``func`` to find out what ``foo`` possibly
     is.
     """
-    funcdef = function_value.tree_node
-
-    if not settings.dynamic_params:
+    if not function_value.inference_state.do_dynamic_params_search:
         return NO_VALUES
 
+    funcdef = function_value.tree_node
+
     path = function_value.get_root_context().py__file__()
     if path is not None and is_stdlib_path(path):
         # We don't want to search for references in the stdlib. Usually people
@@ -216,12 +215,10 @@ def _check_name_for_execution(inference_state, context, compare_node, name, trai
                 for name, trailer in potential_nodes:
                     if value_node.start_pos < name.start_pos < value_node.end_pos:
                         random_context = execution_context.create_context(name)
-                        iterator = _check_name_for_execution(
+                        yield from _check_name_for_execution(
                             inference_state,
                             random_context,
                             compare_node,
                             name,
                             trailer
                         )
-                        for arguments in iterator:
-                            yield arguments

jedi/inference/filters.py

@@ -3,23 +3,25 @@ Filters are objects that you can use to filter names in different scopes. They
 are needed for name resolution.
 """
 from abc import abstractmethod
+from typing import List, MutableMapping, Type
 import weakref
 
 from parso.tree import search_ancestor
+from parso.python.tree import Name, UsedNamesMapping
 
-from jedi._compatibility import use_metaclass
 from jedi.inference import flow_analysis
 from jedi.inference.base_value import ValueSet, ValueWrapper, \
     LazyValueWrapper
-from jedi.parser_utils import get_cached_parent_scope
+from jedi.parser_utils import get_cached_parent_scope, get_parso_cache_node
 from jedi.inference.utils import to_list
 from jedi.inference.names import TreeNameDefinition, ParamName, \
-    AnonymousParamName, AbstractNameDefinition
+    AnonymousParamName, AbstractNameDefinition, NameWrapper
 
+_definition_name_cache: MutableMapping[UsedNamesMapping, List[Name]]
 _definition_name_cache = weakref.WeakKeyDictionary()
 
 
-class AbstractFilter(object):
+class AbstractFilter:
     _until_position = None
 
     def _filter(self, names):
@@ -36,8 +38,8 @@ class AbstractFilter(object):
         raise NotImplementedError
 
 
-class FilterWrapper(object):
-    name_wrapper_class = None
+class FilterWrapper:
+    name_wrapper_class: Type[NameWrapper]
 
     def __init__(self, wrapped_filter):
         self._wrapped_filter = wrapped_filter
@@ -52,11 +54,15 @@ class FilterWrapper(object):
         return self.wrap_names(self._wrapped_filter.values())
 
 
-def _get_definition_names(used_names, name_key):
+def _get_definition_names(parso_cache_node, used_names, name_key):
+    if parso_cache_node is None:
+        names = used_names.get(name_key, ())
+        return tuple(name for name in names if name.is_definition(include_setitem=True))
+
     try:
-        for_module = _definition_name_cache[used_names]
+        for_module = _definition_name_cache[parso_cache_node]
     except KeyError:
-        for_module = _definition_name_cache[used_names] = {}
+        for_module = _definition_name_cache[parso_cache_node] = {}
 
     try:
         return for_module[name_key]
@@ -68,31 +74,51 @@ def _get_definition_names(used_names, name_key):
         return result
 
 
-class AbstractUsedNamesFilter(AbstractFilter):
+class _AbstractUsedNamesFilter(AbstractFilter):
     name_class = TreeNameDefinition
 
-    def __init__(self, parent_context, parser_scope):
-        self._parser_scope = parser_scope
-        self._module_node = self._parser_scope.get_root_node()
-        self._used_names = self._module_node.get_used_names()
+    def __init__(self, parent_context, node_context=None):
+        if node_context is None:
+            node_context = parent_context
+        self._node_context = node_context
+        self._parser_scope = node_context.tree_node
+        module_context = node_context.get_root_context()
+        # It is quite hacky that we have to use that. This is for caching
+        # certain things with a WeakKeyDictionary. However, parso intentionally
+        # uses slots (to save memory) and therefore we end up with having to
+        # have a weak reference to the object that caches the tree.
+        #
+        # Previously we have tried to solve this by using a weak reference onto
+        # used_names. However that also does not work, because it has a
+        # reference from the module, which itself is referenced by any node
+        # through parents.
+        path = module_context.py__file__()
+        if path is None:
+            # If the path is None, there is no guarantee that parso caches it.
+            self._parso_cache_node = None
+        else:
+            self._parso_cache_node = get_parso_cache_node(
+                module_context.inference_state.latest_grammar
+                if module_context.is_stub() else module_context.inference_state.grammar,
+                path
+            )
+        self._used_names = module_context.tree_node.get_used_names()
         self.parent_context = parent_context
 
-    def get(self, name, **filter_kwargs):
+    def get(self, name):
         return self._convert_names(self._filter(
-            _get_definition_names(self._used_names, name),
-            **filter_kwargs
+            _get_definition_names(self._parso_cache_node, self._used_names, name),
         ))
 
     def _convert_names(self, names):
         return [self.name_class(self.parent_context, name) for name in names]
 
-    def values(self, **filter_kwargs):
+    def values(self):
         return self._convert_names(
             name
             for name_key in self._used_names
             for name in self._filter(
-                _get_definition_names(self._used_names, name_key),
-                **filter_kwargs
+                _get_definition_names(self._parso_cache_node, self._used_names, name_key),
             )
         )
 
@@ -100,7 +126,7 @@ class AbstractUsedNamesFilter(AbstractFilter):
         return '<%s: %s>' % (self.__class__.__name__, self.parent_context)
 
 
-class ParserTreeFilter(AbstractUsedNamesFilter):
+class ParserTreeFilter(_AbstractUsedNamesFilter):
     def __init__(self, parent_context, node_context=None, until_position=None,
                  origin_scope=None):
         """
@@ -109,15 +135,12 @@ class ParserTreeFilter(AbstractUsedNamesFilter):
         value, but for some type inference it's important to have a local
         value of the other classes.
         """
-        if node_context is None:
-            node_context = parent_context
-        super(ParserTreeFilter, self).__init__(parent_context, node_context.tree_node)
-        self._node_context = node_context
+        super().__init__(parent_context, node_context)
         self._origin_scope = origin_scope
         self._until_position = until_position
 
     def _filter(self, names):
-        names = super(ParserTreeFilter, self)._filter(names)
+        names = super()._filter(names)
         names = [n for n in names if self._is_name_reachable(n)]
         return list(self._check_flows(names))
 
@@ -126,7 +149,7 @@ class ParserTreeFilter(AbstractUsedNamesFilter):
         if parent.type == 'trailer':
             return False
         base_node = parent if parent.type in ('classdef', 'funcdef') else name
-        return get_cached_parent_scope(self._used_names, base_node) == self._parser_scope
+        return get_cached_parent_scope(self._parso_cache_node, base_node) == self._parser_scope
 
     def _check_flows(self, names):
         for name in sorted(names, key=lambda name: name.start_pos, reverse=True):
@@ -145,7 +168,7 @@ class ParserTreeFilter(AbstractUsedNamesFilter):
 
 class _FunctionExecutionFilter(ParserTreeFilter):
     def __init__(self, parent_context, function_value, until_position, origin_scope):
-        super(_FunctionExecutionFilter, self).__init__(
+        super().__init__(
             parent_context,
             until_position=until_position,
             origin_scope=origin_scope,
@@ -169,9 +192,9 @@ class _FunctionExecutionFilter(ParserTreeFilter):
 
 
 class FunctionExecutionFilter(_FunctionExecutionFilter):
-    def __init__(self, *args, **kwargs):
-        self._arguments = kwargs.pop('arguments')  # Python 2
-        super(FunctionExecutionFilter, self).__init__(*args, **kwargs)
+    def __init__(self, *args, arguments, **kwargs):
+        super().__init__(*args, **kwargs)
+        self._arguments = arguments
 
     def _convert_param(self, param, name):
         return ParamName(self._function_value, name, self._arguments)
@@ -182,7 +205,7 @@ class AnonymousFunctionExecutionFilter(_FunctionExecutionFilter):
         return AnonymousParamName(self._function_value, name)
 
 
-class GlobalNameFilter(AbstractUsedNamesFilter):
+class GlobalNameFilter(_AbstractUsedNamesFilter):
     def get(self, name):
         try:
             names = self._used_names[name]
@@ -232,7 +255,7 @@ class DictFilter(AbstractFilter):
         return '<%s: for {%s}>' % (self.__class__.__name__, keys)
 
 
-class MergedFilter(object):
+class MergedFilter:
     def __init__(self, *filters):
         self._filters = filters
 
@@ -248,16 +271,16 @@ class MergedFilter(object):
 
 class _BuiltinMappedMethod(ValueWrapper):
     """``Generator.__next__`` ``dict.values`` methods and so on."""
-    api_type = u'function'
+    api_type = 'function'
 
     def __init__(self, value, method, builtin_func):
-        super(_BuiltinMappedMethod, self).__init__(builtin_func)
+        super().__init__(builtin_func)
         self._value = value
         self._method = method
 
     def py__call__(self, arguments):
         # TODO add TypeError if params are given/or not correct.
-        return self._method(self._value)
+        return self._method(self._value, arguments)
 
 
 class SpecialMethodFilter(DictFilter):
@@ -266,14 +289,9 @@ class SpecialMethodFilter(DictFilter):
     classes like Generator (for __next__, etc).
     """
     class SpecialMethodName(AbstractNameDefinition):
-        api_type = u'function'
-
-        def __init__(self, parent_context, string_name, value, builtin_value):
-            callable_, python_version = value
-            if python_version is not None and \
-                    python_version != parent_context.inference_state.environment.version_info.major:
-                raise KeyError
+        api_type = 'function'
 
+        def __init__(self, parent_context, string_name, callable_, builtin_value):
             self.parent_context = parent_context
             self.string_name = string_name
             self._callable = callable_
@@ -295,7 +313,7 @@ class SpecialMethodFilter(DictFilter):
             ])
 
     def __init__(self, value, dct, builtin_value):
-        super(SpecialMethodFilter, self).__init__(dct)
+        super().__init__(dct)
         self.value = value
         self._builtin_value = builtin_value
         """
@@ -311,7 +329,7 @@ class SpecialMethodFilter(DictFilter):
 
 class _OverwriteMeta(type):
     def __init__(cls, name, bases, dct):
-        super(_OverwriteMeta, cls).__init__(name, bases, dct)
+        super().__init__(name, bases, dct)
 
         base_dct = {}
         for base_cls in reversed(cls.__bases__):
@@ -328,28 +346,26 @@ class _OverwriteMeta(type):
         cls.overwritten_methods = base_dct
 
 
-class _AttributeOverwriteMixin(object):
+class _AttributeOverwriteMixin:
     def get_filters(self, *args, **kwargs):
         yield SpecialMethodFilter(self, self.overwritten_methods, self._wrapped_value)
-
-        for filter in self._wrapped_value.get_filters():
-            yield filter
+        yield from self._wrapped_value.get_filters(*args, **kwargs)
 
 
-class LazyAttributeOverwrite(use_metaclass(_OverwriteMeta, _AttributeOverwriteMixin,
-                                           LazyValueWrapper)):
+class LazyAttributeOverwrite(_AttributeOverwriteMixin, LazyValueWrapper,
+                             metaclass=_OverwriteMeta):
     def __init__(self, inference_state):
         self.inference_state = inference_state
 
 
-class AttributeOverwrite(use_metaclass(_OverwriteMeta, _AttributeOverwriteMixin,
-                                       ValueWrapper)):
+class AttributeOverwrite(_AttributeOverwriteMixin, ValueWrapper,
+                         metaclass=_OverwriteMeta):
     pass
 
 
-def publish_method(method_name, python_version_match=None):
+def publish_method(method_name):
     def decorator(func):
         dct = func.__dict__.setdefault('registered_overwritten_methods', {})
-        dct[method_name] = func, python_version_match
+        dct[method_name] = func
         return func
     return decorator

jedi/inference/flow_analysis.py

@@ -1,12 +1,14 @@
+from typing import Dict, Optional
+
 from jedi.parser_utils import get_flow_branch_keyword, is_scope, get_parent_scope
 from jedi.inference.recursion import execution_allowed
 from jedi.inference.helpers import is_big_annoying_library
 
 
-class Status(object):
-    lookup_table = {}
+class Status:
+    lookup_table: Dict[Optional[bool], 'Status'] = {}
 
-    def __init__(self, value, name):
+    def __init__(self, value: Optional[bool], name: str) -> None:
         self._value = value
         self._name = name
         Status.lookup_table[value] = self

jedi/inference/gradual/__init__.py

@@ -0,0 +1,4 @@
+"""
+It is unfortunately not well documented how stubs and annotations work in Jedi.
+If somebody needs an introduction, please let me know.
+"""

jedi/inference/gradual/annotation.py

@@ -6,15 +6,14 @@ as annotations in future python versions.
 """
 
 import re
+from inspect import Parameter
 
 from parso import ParserSyntaxError, parse
 
-from jedi._compatibility import force_unicode, Parameter
 from jedi.inference.cache import inference_state_method_cache
 from jedi.inference.base_value import ValueSet, NO_VALUES
-from jedi.inference.gradual.base import DefineGenericBase, GenericClass
+from jedi.inference.gradual.base import DefineGenericBaseClass, GenericClass
 from jedi.inference.gradual.generics import TupleGenericManager
-from jedi.inference.gradual.typing import TypingClassValueWithIndex
 from jedi.inference.gradual.type_var import TypeVar
 from jedi.inference.helpers import is_string
 from jedi.inference.compiled import builtin_from_name
@@ -54,8 +53,10 @@ def _infer_annotation_string(context, string, index=None):
     value_set = context.infer_node(node)
     if index is not None:
         value_set = value_set.filter(
-            lambda value: value.array_type == u'tuple'  # noqa
-                            and len(list(value.py__iter__())) >= index
+            lambda value: (
+                value.array_type == 'tuple'
+                and len(list(value.py__iter__())) >= index
+            )
         ).py__simple_getitem__(index)
     return value_set
 
@@ -63,7 +64,7 @@ def _infer_annotation_string(context, string, index=None):
 def _get_forward_reference_node(context, string):
     try:
         new_node = context.inference_state.grammar.parse(
-            force_unicode(string),
+            string,
             start_symbol='eval_input',
             error_recovery=False
         )
@@ -111,7 +112,7 @@ def _split_comment_param_declaration(decl_text):
 @inference_state_method_cache()
 def infer_param(function_value, param, ignore_stars=False):
     values = _infer_param(function_value, param)
-    if ignore_stars:
+    if ignore_stars or not values:
         return values
     inference_state = function_value.inference_state
     if param.star_count == 1:
@@ -119,7 +120,7 @@ def infer_param(function_value, param, ignore_stars=False):
         return ValueSet([GenericClass(
             tuple_,
             TupleGenericManager((values,)),
-        ) for c in values])
+        )])
     elif param.star_count == 2:
         dct = builtin_from_name(inference_state, 'dict')
         generics = (
@@ -129,8 +130,7 @@ def infer_param(function_value, param, ignore_stars=False):
         return ValueSet([GenericClass(
             dct,
             TupleGenericManager(generics),
-        ) for c in values])
-        pass
+        )])
     return values
 
 
@@ -140,8 +140,7 @@ def _infer_param(function_value, param):
     """
     annotation = param.annotation
     if annotation is None:
-        # If no Python 3-style annotation, look for a Python 2-style comment
-        # annotation.
+        # If no Python 3-style annotation, look for a comment annotation.
         # Identify parameters to function in the same sequence as they would
         # appear in a type comment.
         all_params = [child for child in param.parent.children
@@ -197,16 +196,47 @@ def py__annotations__(funcdef):
     return dct
 
 
+def resolve_forward_references(context, all_annotations):
+    def resolve(node):
+        if node is None or node.type != 'string':
+            return node
+
+        node = _get_forward_reference_node(
+            context,
+            context.inference_state.compiled_subprocess.safe_literal_eval(
+                node.value,
+            ),
+        )
+
+        if node is None:
+            # There was a string, but it's not a valid annotation
+            return None
+
+        # The forward reference tree has an additional root node ('eval_input')
+        # that we don't want. Extract the node we do want, that is equivalent to
+        # the nodes returned by `py__annotations__` for a non-quoted node.
+        node = node.children[0]
+
+        return node
+
+    return {name: resolve(node) for name, node in all_annotations.items()}
+
+
 @inference_state_method_cache()
 def infer_return_types(function, arguments):
     """
     Infers the type of a function's return value,
     according to type annotations.
     """
-    all_annotations = py__annotations__(function.tree_node)
+    context = function.get_default_param_context()
+    all_annotations = resolve_forward_references(
+        context,
+        py__annotations__(function.tree_node),
+    )
     annotation = all_annotations.get("return", None)
     if annotation is None:
-        # If there is no Python 3-type annotation, look for a Python 2-type annotation
+        # If there is no Python 3-type annotation, look for an annotation
+        # comment.
         node = function.tree_node
         comment = parser_utils.get_following_comment_same_line(node)
         if comment is None:
@@ -217,13 +247,10 @@ def infer_return_types(function, arguments):
             return NO_VALUES
 
         return _infer_annotation_string(
-            function.get_default_param_context(),
+            context,
             match.group(1).strip()
         ).execute_annotation()
-        if annotation is None:
-            return NO_VALUES
 
-    context = function.get_default_param_context()
     unknown_type_vars = find_unknown_type_vars(context, annotation)
     annotation_values = infer_annotation(context, annotation)
     if not unknown_type_vars:
@@ -233,7 +260,7 @@ def infer_return_types(function, arguments):
 
     return ValueSet.from_sets(
         ann.define_generics(type_var_dict)
-        if isinstance(ann, (DefineGenericBase, TypeVar)) else ValueSet({ann})
+        if isinstance(ann, (DefineGenericBaseClass, TypeVar)) else ValueSet({ann})
         for ann in annotation_values
     ).execute_annotation()
 
@@ -269,29 +296,29 @@ def infer_type_vars_for_execution(function, arguments, annotation_dict):
             elif kind is Parameter.VAR_KEYWORD:
                 # TODO _dict_values is not public.
                 actual_value_set = actual_value_set.try_merge('_dict_values')
-            for ann in annotation_value_set:
-                _merge_type_var_dicts(
-                    annotation_variable_results,
-                    _infer_type_vars(ann, actual_value_set),
-                )
+            merge_type_var_dicts(
+                annotation_variable_results,
+                annotation_value_set.infer_type_vars(actual_value_set),
+            )
     return annotation_variable_results
 
 
 def infer_return_for_callable(arguments, param_values, result_values):
-    result = NO_VALUES
+    all_type_vars = {}
     for pv in param_values:
         if pv.array_type == 'list':
-            type_var_dict = infer_type_vars_for_callable(arguments, pv.py__iter__())
+            type_var_dict = _infer_type_vars_for_callable(arguments, pv.py__iter__())
+            all_type_vars.update(type_var_dict)
 
-            result |= ValueSet.from_sets(
-                v.define_generics(type_var_dict)
-                if isinstance(v, (DefineGenericBase, TypeVar)) else ValueSet({v})
-                for v in result_values
-            ).execute_annotation()
-    return result
+    return ValueSet.from_sets(
+        v.define_generics(all_type_vars)
+        if isinstance(v, (DefineGenericBaseClass, TypeVar))
+        else ValueSet({v})
+        for v in result_values
+    ).execute_annotation()
 
 
-def infer_type_vars_for_callable(arguments, lazy_params):
+def _infer_type_vars_for_callable(arguments, lazy_params):
     """
     Infers type vars for the Calllable class:
 
@@ -302,15 +329,14 @@ def infer_type_vars_for_callable(arguments, lazy_params):
         callable_param_values = lazy_callable_param.infer()
         # Infer unknown type var
         actual_value_set = lazy_value.infer()
-        for v in callable_param_values:
-            _merge_type_var_dicts(
-                annotation_variable_results,
-                _infer_type_vars(v, actual_value_set),
-            )
+        merge_type_var_dicts(
+            annotation_variable_results,
+            callable_param_values.infer_type_vars(actual_value_set),
+        )
     return annotation_variable_results
 
 
-def _merge_type_var_dicts(base_dict, new_dict):
+def merge_type_var_dicts(base_dict, new_dict):
     for type_var_name, values in new_dict.items():
         if values:
             try:
@@ -319,88 +345,55 @@ def _merge_type_var_dicts(base_dict, new_dict):
                 base_dict[type_var_name] = values
 
 
-def _infer_type_vars(annotation_value, value_set, is_class_value=False):
+def merge_pairwise_generics(annotation_value, annotated_argument_class):
     """
-    This function tries to find information about undefined type vars and
-    returns a dict from type var name to value set.
+    Match up the generic parameters from the given argument class to the
+    target annotation.
 
-    This is for example important to understand what `iter([1])` returns.
-    According to typeshed, `iter` returns an `Iterator[_T]`:
+    This walks the generic parameters immediately within the annotation and
+    argument's type, in order to determine the concrete values of the
+    annotation's parameters for the current case.
 
-        def iter(iterable: Iterable[_T]) -> Iterator[_T]: ...
+    For example, given the following code:
 
-    This functions would generate `int` for `_T` in this case, because it
-    unpacks the `Iterable`.
+        def values(mapping: Mapping[K, V]) -> List[V]: ...
+
+        for val in values({1: 'a'}):
+            val
+
+    Then this function should be given representations of `Mapping[K, V]`
+    and `Mapping[int, str]`, so that it can determine that `K` is `int and
+    `V` is `str`.
+
+    Note that it is responsibility of the caller to traverse the MRO of the
+    argument type as needed in order to find the type matching the
+    annotation (in this case finding `Mapping[int, str]` as a parent of
+    `Dict[int, str]`).
+
+    Parameters
+    ----------
+
+    `annotation_value`: represents the annotation to infer the concrete
+        parameter types of.
+
+    `annotated_argument_class`: represents the annotated class of the
+        argument being passed to the object annotated by `annotation_value`.
     """
+
     type_var_dict = {}
-    if isinstance(annotation_value, TypeVar):
-        if not is_class_value:
-            return {annotation_value.py__name__(): value_set.py__class__()}
-        return {annotation_value.py__name__(): value_set}
-    elif isinstance(annotation_value, TypingClassValueWithIndex):
-        name = annotation_value.py__name__()
-        if name == 'Type':
-            given = annotation_value.get_generics()
-            if given:
-                for nested_annotation_value in given[0]:
-                    _merge_type_var_dicts(
-                        type_var_dict,
-                        _infer_type_vars(
-                            nested_annotation_value,
-                            value_set,
-                            is_class_value=True,
-                        )
-                    )
-        elif name == 'Callable':
-            given = annotation_value.get_generics()
-            if len(given) == 2:
-                for nested_annotation_value in given[1]:
-                    _merge_type_var_dicts(
-                        type_var_dict,
-                        _infer_type_vars(
-                            nested_annotation_value,
-                            value_set.execute_annotation(),
-                        )
-                    )
-    elif isinstance(annotation_value, GenericClass):
-        name = annotation_value.py__name__()
-        if name == 'Iterable':
-            given = annotation_value.get_generics()
-            if given:
-                for nested_annotation_value in given[0]:
-                    _merge_type_var_dicts(
-                        type_var_dict,
-                        _infer_type_vars(
-                            nested_annotation_value,
-                            value_set.merge_types_of_iterate()
-                        )
-                    )
-        elif name == 'Mapping':
-            given = annotation_value.get_generics()
-            if len(given) == 2:
-                for value in value_set:
-                    try:
-                        method = value.get_mapping_item_values
-                    except AttributeError:
-                        continue
-                    key_values, value_values = method()
 
-                    for nested_annotation_value in given[0]:
-                        _merge_type_var_dicts(
-                            type_var_dict,
-                            _infer_type_vars(
-                                nested_annotation_value,
-                                key_values,
-                            )
-                        )
-                    for nested_annotation_value in given[1]:
-                        _merge_type_var_dicts(
-                            type_var_dict,
-                            _infer_type_vars(
-                                nested_annotation_value,
-                                value_values,
-                            )
-                        )
+    if not isinstance(annotated_argument_class, DefineGenericBaseClass):
+        return type_var_dict
+
+    annotation_generics = annotation_value.get_generics()
+    actual_generics = annotated_argument_class.get_generics()
+
+    for annotation_generics_set, actual_generic_set in zip(annotation_generics, actual_generics):
+        merge_type_var_dicts(
+            type_var_dict,
+            annotation_generics_set.infer_type_vars(actual_generic_set.execute_annotation()),
+        )
+
     return type_var_dict
 
 
@@ -409,6 +402,10 @@ def find_type_from_comment_hint_for(context, node, name):
 
 
 def find_type_from_comment_hint_with(context, node, name):
+    if len(node.children) > 4:
+        # In case there are multiple with_items, we do not want a type hint for
+        # now.
+        return []
     assert len(node.children[1].children) == 3, \
         "Can only be here when children[1] is 'foo() as f'"
     varlist = node.children[1].children[2]

jedi/inference/gradual/base.py

@@ -23,10 +23,9 @@ class _BoundTypeVarName(AbstractNameDefinition):
         def iter_():
             for value in self._value_set:
                 # Replace any with the constraints if they are there.
-                from jedi.inference.gradual.typing import Any
-                if isinstance(value, Any):
-                    for constraint in self._type_var.constraints:
-                        yield constraint
+                from jedi.inference.gradual.typing import AnyClass
+                if isinstance(value, AnyClass):
+                    yield from self._type_var.constraints
                 else:
                     yield value
         return ValueSet(iter_())
@@ -38,7 +37,7 @@ class _BoundTypeVarName(AbstractNameDefinition):
         return '<%s %s -> %s>' % (self.__class__.__name__, self.py__name__(), self._value_set)
 
 
-class _TypeVarFilter(object):
+class _TypeVarFilter:
     """
     A filter for all given variables in a class.
 
@@ -70,18 +69,17 @@ class _TypeVarFilter(object):
 
 class _AnnotatedClassContext(ClassContext):
     def get_filters(self, *args, **kwargs):
-        filters = super(_AnnotatedClassContext, self).get_filters(
+        filters = super().get_filters(
             *args, **kwargs
         )
-        for f in filters:
-            yield f
+        yield from filters
 
         # The type vars can only be looked up if it's a global search and
         # not a direct lookup on the class.
         yield self._value.get_type_var_filter()
 
 
-class DefineGenericBase(LazyValueWrapper):
+class DefineGenericBaseClass(LazyValueWrapper):
     def __init__(self, generics_manager):
         self._generics_manager = generics_manager
 
@@ -99,7 +97,7 @@ class DefineGenericBase(LazyValueWrapper):
         for generic_set in self.get_generics():
             values = NO_VALUES
             for generic in generic_set:
-                if isinstance(generic, (GenericClass, TypeVar)):
+                if isinstance(generic, (DefineGenericBaseClass, TypeVar)):
                     result = generic.define_generics(type_var_dict)
                     values |= result
                     if result != ValueSet({generic}):
@@ -119,7 +117,7 @@ class DefineGenericBase(LazyValueWrapper):
         )])
 
     def is_same_class(self, other):
-        if not isinstance(other, DefineGenericBase):
+        if not isinstance(other, DefineGenericBaseClass):
             return False
 
         if self.tree_node != other.tree_node:
@@ -138,11 +136,19 @@ class DefineGenericBase(LazyValueWrapper):
             any(
                 # TODO why is this ordering the correct one?
                 cls2.is_same_class(cls1)
-                for cls1 in class_set1
-                for cls2 in class_set2
+                # TODO I'm still not sure gather_annotation_classes is a good
+                # idea. They are essentially here to avoid comparing Tuple <=>
+                # tuple and instead compare tuple <=> tuple, but at the moment
+                # the whole `is_same_class` and `is_sub_class` matching is just
+                # not in the best shape.
+                for cls1 in class_set1.gather_annotation_classes()
+                for cls2 in class_set2.gather_annotation_classes()
             ) for class_set1, class_set2 in zip(given_params1, given_params2)
         )
 
+    def get_signatures(self):
+        return []
+
     def __repr__(self):
         return '<%s: %s%s>' % (
             self.__class__.__name__,
@@ -151,7 +157,7 @@ class DefineGenericBase(LazyValueWrapper):
         )
 
 
-class GenericClass(ClassMixin, DefineGenericBase):
+class GenericClass(DefineGenericBaseClass, ClassMixin):
     """
     A class that is defined with generics, might be something simple like:
 
@@ -159,17 +165,29 @@ class GenericClass(ClassMixin, DefineGenericBase):
         my_foo_int_cls = Foo[int]
     """
     def __init__(self, class_value, generics_manager):
-        super(GenericClass, self).__init__(generics_manager)
+        super().__init__(generics_manager)
         self._class_value = class_value
 
     def _get_wrapped_value(self):
         return self._class_value
 
+    def get_type_hint(self, add_class_info=True):
+        n = self.py__name__()
+        # Not sure if this is the best way to do this, but all of these types
+        # are a bit special in that they have type aliases and other ways to
+        # become lower case. It's probably better to make them upper case,
+        # because that's what you can use in annotations.
+        n = dict(list="List", dict="Dict", set="Set", tuple="Tuple").get(n, n)
+        s = n + self._generics_manager.get_type_hint()
+        if add_class_info:
+            return 'Type[%s]' % s
+        return s
+
     def get_type_var_filter(self):
         return _TypeVarFilter(self.get_generics(), self.list_type_vars())
 
     def py__call__(self, arguments):
-        instance, = super(GenericClass, self).py__call__(arguments)
+        instance, = super().py__call__(arguments)
         return ValueSet([_GenericInstanceWrapper(instance)])
 
     def _as_context(self):
@@ -178,21 +196,64 @@ class GenericClass(ClassMixin, DefineGenericBase):
     @to_list
     def py__bases__(self):
         for base in self._wrapped_value.py__bases__():
-            yield _LazyGenericBaseClass(self, base)
+            yield _LazyGenericBaseClass(self, base, self._generics_manager)
 
     def _create_instance_with_generics(self, generics_manager):
         return GenericClass(self._class_value, generics_manager)
 
     def is_sub_class_of(self, class_value):
-        if super(GenericClass, self).is_sub_class_of(class_value):
+        if super().is_sub_class_of(class_value):
             return True
         return self._class_value.is_sub_class_of(class_value)
 
+    def with_generics(self, generics_tuple):
+        return self._class_value.with_generics(generics_tuple)
 
-class _LazyGenericBaseClass(object):
-    def __init__(self, class_value, lazy_base_class):
+    def infer_type_vars(self, value_set):
+        # Circular
+        from jedi.inference.gradual.annotation import merge_pairwise_generics, merge_type_var_dicts
+
+        annotation_name = self.py__name__()
+        type_var_dict = {}
+        if annotation_name == 'Iterable':
+            annotation_generics = self.get_generics()
+            if annotation_generics:
+                return annotation_generics[0].infer_type_vars(
+                    value_set.merge_types_of_iterate(),
+                )
+        else:
+            # Note: we need to handle the MRO _in order_, so we need to extract
+            # the elements from the set first, then handle them, even if we put
+            # them back in a set afterwards.
+            for py_class in value_set:
+                if py_class.is_instance() and not py_class.is_compiled():
+                    py_class = py_class.get_annotated_class_object()
+                else:
+                    continue
+
+                if py_class.api_type != 'class':
+                    # Functions & modules don't have an MRO and we're not
+                    # expecting a Callable (those are handled separately within
+                    # TypingClassValueWithIndex).
+                    continue
+
+                for parent_class in py_class.py__mro__():
+                    class_name = parent_class.py__name__()
+                    if annotation_name == class_name:
+                        merge_type_var_dicts(
+                            type_var_dict,
+                            merge_pairwise_generics(self, parent_class),
+                        )
+                        break
+
+        return type_var_dict
+
+
+class _LazyGenericBaseClass:
+    def __init__(self, class_value, lazy_base_class, generics_manager):
         self._class_value = class_value
         self._lazy_base_class = lazy_base_class
+        self._generics_manager = generics_manager
 
     @iterator_to_value_set
     def infer(self):
@@ -205,7 +266,17 @@ class _LazyGenericBaseClass(object):
                     TupleGenericManager(tuple(self._remap_type_vars(base))),
                 )
             else:
-                yield base
+                if base.is_class_mixin():
+                    # This case basically allows classes like `class Foo(List)`
+                    # to be used like `Foo[int]`. The generics are not
+                    # necessary and can be used later.
+                    yield GenericClass.create_cached(
+                        base.inference_state,
+                        base,
+                        self._generics_manager,
+                    )
+                else:
+                    yield base
 
     def _remap_type_vars(self, base):
         from jedi.inference.gradual.type_var import TypeVar
@@ -225,6 +296,9 @@ class _LazyGenericBaseClass(object):
                     new |= ValueSet([type_var])
             yield new
 
+    def __repr__(self):
+        return '<%s: %s>' % (self.__class__.__name__, self._lazy_base_class)
+
 
 class _GenericInstanceWrapper(ValueWrapper):
     def py__stop_iteration_returns(self):
@@ -236,9 +310,12 @@ class _GenericInstanceWrapper(ValueWrapper):
                 except IndexError:
                     pass
             elif cls.py__name__() == 'Iterator':
-                return ValueSet([builtin_from_name(self.inference_state, u'None')])
+                return ValueSet([builtin_from_name(self.inference_state, 'None')])
         return self._wrapped_value.py__stop_iteration_returns()
 
+    def get_type_hint(self, add_class_info=True):
+        return self._wrapped_value.class_value.get_type_hint(add_class_info=False)
+
 
 class _PseudoTreeNameClass(Value):
     """
@@ -250,8 +327,10 @@ class _PseudoTreeNameClass(Value):
     this class. Essentially this class makes it possible to goto that `Tuple`
     name, without affecting anything else negatively.
     """
+    api_type = 'class'
+
     def __init__(self, parent_context, tree_name):
-        super(_PseudoTreeNameClass, self).__init__(
+        super().__init__(
             parent_context.inference_state,
             parent_context
         )
@@ -276,15 +355,17 @@ class _PseudoTreeNameClass(Value):
         yield EmptyFilter()
 
     def py__class__(self):
-        # TODO this is obviously not correct, but at least gives us a class if
-        # we have none. Some of these objects don't really have a base class in
-        # typeshed.
-        return builtin_from_name(self.inference_state, u'object')
+        # This might not be 100% correct, but it is good enough. The details of
+        # the typing library are not really an issue for Jedi.
+        return builtin_from_name(self.inference_state, 'type')
 
     @property
     def name(self):
         return ValueName(self, self._tree_name)
 
+    def get_qualified_names(self):
+        return (self._tree_name.value,)
+
     def __repr__(self):
         return '%s(%s)' % (self.__class__.__name__, self._tree_name.value)
 
@@ -302,13 +383,16 @@ class BaseTypingValue(LazyValueWrapper):
     def _get_wrapped_value(self):
         return _PseudoTreeNameClass(self.parent_context, self._tree_name)
 
+    def get_signatures(self):
+        return self._wrapped_value.get_signatures()
+
     def __repr__(self):
         return '%s(%s)' % (self.__class__.__name__, self._tree_name.value)
 
 
-class BaseTypingValueWithGenerics(DefineGenericBase):
+class BaseTypingClassWithGenerics(DefineGenericBaseClass):
     def __init__(self, parent_context, tree_name, generics_manager):
-        super(BaseTypingValueWithGenerics, self).__init__(generics_manager)
+        super().__init__(generics_manager)
         self.inference_state = parent_context.inference_state
         self.parent_context = parent_context
         self._tree_name = tree_name
@@ -319,3 +403,32 @@ class BaseTypingValueWithGenerics(DefineGenericBase):
     def __repr__(self):
         return '%s(%s%s)' % (self.__class__.__name__, self._tree_name.value,
                              self._generics_manager)
+
+
+class BaseTypingInstance(LazyValueWrapper):
+    def __init__(self, parent_context, class_value, tree_name, generics_manager):
+        self.inference_state = class_value.inference_state
+        self.parent_context = parent_context
+        self._class_value = class_value
+        self._tree_name = tree_name
+        self._generics_manager = generics_manager
+
+    def py__class__(self):
+        return self._class_value
+
+    def get_annotated_class_object(self):
+        return self._class_value
+
+    def get_qualified_names(self):
+        return (self.py__name__(),)
+
+    @property
+    def name(self):
+        return ValueName(self, self._tree_name)
+
+    def _get_wrapped_value(self):
+        object_, = builtin_from_name(self.inference_state, 'object').execute_annotation()
+        return object_
+
+    def __repr__(self):
+        return '<%s: %s>' % (self.__class__.__name__, self._generics_manager)

jedi/inference/gradual/conversion.py

@@ -3,6 +3,8 @@ from jedi.inference.base_value import ValueSet, \
     NO_VALUES
 from jedi.inference.utils import to_list
 from jedi.inference.gradual.stub_value import StubModuleValue
+from jedi.inference.gradual.typeshed import try_to_load_stub_cached
+from jedi.inference.value.decorator import Decoratee
 
 
 def _stub_to_python_value_set(stub_value, ignore_compiled=False):
@@ -10,8 +12,13 @@ def _stub_to_python_value_set(stub_value, ignore_compiled=False):
     if not stub_module_context.is_stub():
         return ValueSet([stub_value])
 
+    decorates = None
+    if isinstance(stub_value, Decoratee):
+        decorates = stub_value._original_value
+
     was_instance = stub_value.is_instance()
     if was_instance:
+        arguments = getattr(stub_value, '_arguments', None)
         stub_value = stub_value.py__class__()
 
     qualified_names = stub_value.get_qualified_names()
@@ -24,11 +31,12 @@ def _stub_to_python_value_set(stub_value, ignore_compiled=False):
         method_name = qualified_names[-1]
         qualified_names = qualified_names[:-1]
         was_instance = True
+        arguments = None
 
     values = _infer_from_stub(stub_module_context, qualified_names, ignore_compiled)
     if was_instance:
         values = ValueSet.from_sets(
-            c.execute_with_values()
+            c.execute_with_values() if arguments is None else c.execute(arguments)
             for c in values
             if c.is_class()
         )
@@ -36,6 +44,8 @@ def _stub_to_python_value_set(stub_value, ignore_compiled=False):
         # Now that the instance has been properly created, we can simply get
         # the method.
         values = values.py__getattribute__(method_name)
+    if decorates is not None:
+        values = ValueSet(Decoratee(v, decorates) for v in values)
     return values
 
 
@@ -73,7 +83,13 @@ def _try_stub_to_python_names(names, prefer_stub_to_compiled=False):
                     converted_names = converted.goto(name.get_public_name())
                     if converted_names:
                         for n in converted_names:
-                            yield n
+                            if n.get_root_context().is_stub():
+                                # If it's a stub again, it means we're going in
+                                # a circle. Probably some imports make it a
+                                # stub again.
+                                yield name
+                            else:
+                                yield n
                         continue
         yield name
 
@@ -81,8 +97,7 @@ def _try_stub_to_python_names(names, prefer_stub_to_compiled=False):
 def _load_stub_module(module):
     if module.is_stub():
         return module
-    from jedi.inference.gradual.typeshed import _try_to_load_stub_cached
-    return _try_to_load_stub_cached(
+    return try_to_load_stub_cached(
         module.inference_state,
         import_names=module.string_names,
         python_value_set=ValueSet([module]),
@@ -120,8 +135,7 @@ def _python_to_stub_names(names, fallback_to_python=False):
                 if converted:
                     converted_names = converted.goto(name.get_public_name())
                     if converted_names:
-                        for n in converted_names:
-                            yield n
+                        yield from converted_names
                         continue
         if fallback_to_python:
             # This is the part where if we haven't found anything, just return

jedi/inference/gradual/generics.py

@@ -23,7 +23,7 @@ def _resolve_forward_references(context, value_set):
             yield value
 
 
-class _AbstractGenericManager(object):
+class _AbstractGenericManager:
     def get_index_and_execute(self, index):
         try:
             return self[index].execute_annotation()
@@ -31,6 +31,9 @@ class _AbstractGenericManager(object):
             debug.warning('No param #%s found for annotation %s', index, self)
             return NO_VALUES
 
+    def get_type_hint(self):
+        return '[%s]' % ', '.join(t.get_type_hint(add_class_info=False) for t in self.to_tuple())
+
 
 class LazyGenericManager(_AbstractGenericManager):
     def __init__(self, context_of_index, index_value):

jedi/inference/gradual/stub_value.py

@@ -10,7 +10,7 @@ class StubModuleValue(ModuleValue):
     _module_name_class = StubModuleName
 
     def __init__(self, non_stub_value_set, *args, **kwargs):
-        super(StubModuleValue, self).__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
         self.non_stub_value_set = non_stub_value_set
 
     def is_stub(self):
@@ -30,7 +30,7 @@ class StubModuleValue(ModuleValue):
                 pass
             else:
                 names.update(method())
-        names.update(super(StubModuleValue, self).sub_modules_dict())
+        names.update(super().sub_modules_dict())
         return names
 
     def _get_stub_filters(self, origin_scope):
@@ -40,14 +40,11 @@ class StubModuleValue(ModuleValue):
         )] + list(self.iter_star_filters())
 
     def get_filters(self, origin_scope=None):
-        filters = super(StubModuleValue, self).get_filters(origin_scope)
-        next(filters)  # Ignore the first filter and replace it with our own
+        filters = super().get_filters(origin_scope)
+        next(filters, None)  # Ignore the first filter and replace it with our own
         stub_filters = self._get_stub_filters(origin_scope=origin_scope)
-        for f in stub_filters:
-            yield f
-
-        for f in filters:
-            yield f
+        yield from stub_filters
+        yield from filters
 
     def _as_context(self):
         return StubModuleContext(self)
@@ -57,15 +54,16 @@ class StubModuleContext(ModuleContext):
     def get_filters(self, until_position=None, origin_scope=None):
         # Make sure to ignore the position, because positions are not relevant
         # for stubs.
-        return super(StubModuleContext, self).get_filters(origin_scope=origin_scope)
+        return super().get_filters(origin_scope=origin_scope)
 
 
 class TypingModuleWrapper(StubModuleValue):
     def get_filters(self, *args, **kwargs):
-        filters = super(TypingModuleWrapper, self).get_filters(*args, **kwargs)
-        yield TypingModuleFilterWrapper(next(filters))
-        for f in filters:
-            yield f
+        filters = super().get_filters(*args, **kwargs)
+        f = next(filters, None)
+        assert f is not None
+        yield TypingModuleFilterWrapper(f)
+        yield from filters
 
     def _as_context(self):
         return TypingModuleContext(self)
@@ -73,22 +71,23 @@ class TypingModuleWrapper(StubModuleValue):
 
 class TypingModuleContext(ModuleContext):
     def get_filters(self, *args, **kwargs):
-        filters = super(TypingModuleContext, self).get_filters(*args, **kwargs)
-        yield TypingModuleFilterWrapper(next(filters))
-        for f in filters:
-            yield f
+        filters = super().get_filters(*args, **kwargs)
+        yield TypingModuleFilterWrapper(next(filters, None))
+        yield from filters
 
 
 class StubFilter(ParserTreeFilter):
     name_class = StubName
 
     def _is_name_reachable(self, name):
-        if not super(StubFilter, self)._is_name_reachable(name):
+        if not super()._is_name_reachable(name):
             return False
 
         # Imports in stub files are only public if they have an "as"
         # export.
         definition = name.get_definition()
+        if definition is None:
+            return False
         if definition.type in ('import_from', 'import_name'):
             if name.parent.type not in ('import_as_name', 'dotted_as_name'):
                 return False

jedi/inference/gradual/type_var.py

@@ -1,10 +1,9 @@
-from jedi._compatibility import unicode, force_unicode
 from jedi import debug
-from jedi.inference.base_value import ValueSet, NO_VALUES
+from jedi.inference.base_value import ValueSet, NO_VALUES, ValueWrapper
 from jedi.inference.gradual.base import BaseTypingValue
 
 
-class TypeVarClass(BaseTypingValue):
+class TypeVarClass(ValueWrapper):
     def py__call__(self, arguments):
         unpacked = arguments.unpack()
 
@@ -18,9 +17,9 @@ class TypeVarClass(BaseTypingValue):
         return ValueSet([TypeVar.create_cached(
             self.inference_state,
             self.parent_context,
-            self._tree_name,
-            var_name,
-            unpacked
+            tree_name=self.tree_node.name,
+            var_name=var_name,
+            unpacked_args=unpacked,
         )])
 
     def _find_string_name(self, lazy_value):
@@ -40,17 +39,14 @@ class TypeVarClass(BaseTypingValue):
             return None
         else:
             safe_value = method(default=None)
-            if self.inference_state.environment.version_info.major == 2:
-                if isinstance(safe_value, bytes):
-                    return force_unicode(safe_value)
-            if isinstance(safe_value, (str, unicode)):
+            if isinstance(safe_value, str):
                 return safe_value
             return None
 
 
 class TypeVar(BaseTypingValue):
     def __init__(self, parent_context, tree_name, var_name, unpacked_args):
-        super(TypeVar, self).__init__(parent_context, tree_name)
+        super().__init__(parent_context, tree_name)
         self._var_name = var_name
 
         self._constraints_lazy_values = []
@@ -102,10 +98,30 @@ class TypeVar(BaseTypingValue):
         else:
             if found:
                 return found
-        return self._get_classes() or ValueSet({self})
+        return ValueSet({self})
 
     def execute_annotation(self):
         return self._get_classes().execute_annotation()
 
+    def infer_type_vars(self, value_set):
+        def iterate():
+            for v in value_set:
+                cls = v.py__class__()
+                if v.is_function() or v.is_class():
+                    cls = TypeWrapper(cls, v)
+                yield cls
+
+        annotation_name = self.py__name__()
+        return {annotation_name: ValueSet(iterate())}
+
     def __repr__(self):
         return '<%s: %s>' % (self.__class__.__name__, self.py__name__())
+
+
+class TypeWrapper(ValueWrapper):
+    def __init__(self, wrapped_value, original_value):
+        super().__init__(wrapped_value)
+        self._original_value = original_value
+
+    def execute_annotation(self):
+        return ValueSet({self._original_value})

jedi/inference/gradual/typeshed.py

@@ -1,74 +1,79 @@
 import os
 import re
 from functools import wraps
+from collections import namedtuple
+from typing import Dict, Mapping, Tuple
+from pathlib import Path
 
+from jedi import settings
 from jedi.file_io import FileIO
-from jedi._compatibility import FileNotFoundError, cast_path
 from jedi.parser_utils import get_cached_code_lines
 from jedi.inference.base_value import ValueSet, NO_VALUES
 from jedi.inference.gradual.stub_value import TypingModuleWrapper, StubModuleValue
 from jedi.inference.value import ModuleValue
 
-_jedi_path = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
-TYPESHED_PATH = os.path.join(_jedi_path, 'third_party', 'typeshed')
+_jedi_path = Path(__file__).parent.parent.parent
+TYPESHED_PATH = _jedi_path.joinpath('third_party', 'typeshed')
+DJANGO_INIT_PATH = _jedi_path.joinpath('third_party', 'django-stubs',
+                                       'django-stubs', '__init__.pyi')
 
 _IMPORT_MAP = dict(
     _collections='collections',
     _socket='socket',
 )
 
+PathInfo = namedtuple('PathInfo', 'path is_third_party')
 
-def _merge_create_stub_map(directories):
+
+def _merge_create_stub_map(path_infos):
     map_ = {}
-    for directory in directories:
-        map_.update(_create_stub_map(directory))
+    for directory_path_info in path_infos:
+        map_.update(_create_stub_map(directory_path_info))
     return map_
 
 
-def _create_stub_map(directory):
+def _create_stub_map(directory_path_info):
     """
     Create a mapping of an importable name in Python to a stub file.
     """
     def generate():
         try:
-            listed = os.listdir(directory)
-        except (FileNotFoundError, OSError):
-            # OSError is Python 2
+            listed = os.listdir(directory_path_info.path)
+        except (FileNotFoundError, NotADirectoryError):
             return
 
         for entry in listed:
-            entry = cast_path(entry)
-            path = os.path.join(directory, entry)
+            path = os.path.join(directory_path_info.path, entry)
             if os.path.isdir(path):
                 init = os.path.join(path, '__init__.pyi')
                 if os.path.isfile(init):
-                    yield entry, init
+                    yield entry, PathInfo(init, directory_path_info.is_third_party)
             elif entry.endswith('.pyi') and os.path.isfile(path):
                 name = entry[:-4]
                 if name != '__init__':
-                    yield name, path
+                    yield name, PathInfo(path, directory_path_info.is_third_party)
 
     # Create a dictionary from the tuple generator.
     return dict(generate())
 
 
 def _get_typeshed_directories(version_info):
-    check_version_list = ['2and3', str(version_info.major)]
+    check_version_list = ['2and3', '3']
     for base in ['stdlib', 'third_party']:
-        base = os.path.join(TYPESHED_PATH, base)
-        base_list = os.listdir(base)
+        base_path = TYPESHED_PATH.joinpath(base)
+        base_list = os.listdir(base_path)
         for base_list_entry in base_list:
             match = re.match(r'(\d+)\.(\d+)$', base_list_entry)
             if match is not None:
-                if int(match.group(1)) == version_info.major \
-                        and int(match.group(2)) <= version_info.minor:
+                if match.group(1) == '3' and int(match.group(2)) <= version_info.minor:
                     check_version_list.append(base_list_entry)
 
         for check_version in check_version_list:
-            yield os.path.join(base, check_version)
+            is_third_party = base != 'stdlib'
+            yield PathInfo(str(base_path.joinpath(check_version)), is_third_party)
 
 
-_version_cache = {}
+_version_cache: Dict[Tuple[int, int], Mapping[str, PathInfo]] = {}
 
 
 def _cache_stub_file_map(version_info):
@@ -102,12 +107,9 @@ def import_module_decorator(func):
                 # ``os.path``, because it's a very important one in Python
                 # that is being achieved by messing with ``sys.modules`` in
                 # ``os``.
-                python_parent = next(iter(parent_module_values))
-                if python_parent is None:
-                    python_parent, = inference_state.import_module(('os',), prefer_stubs=False)
                 python_value_set = ValueSet.from_sets(
                     func(inference_state, (n,), None, sys_path,)
-                    for n in [u'posixpath', u'ntpath', u'macpath', u'os2emxpath']
+                    for n in ['posixpath', 'ntpath', 'macpath', 'os2emxpath']
                 )
             else:
                 python_value_set = ValueSet.from_sets(
@@ -116,11 +118,11 @@ def import_module_decorator(func):
                 )
             inference_state.module_cache.add(import_names, python_value_set)
 
-        if not prefer_stubs:
+        if not prefer_stubs or import_names[0] in settings.auto_import_modules:
             return python_value_set
 
-        stub = _try_to_load_stub_cached(inference_state, import_names, python_value_set,
-                                        parent_module_value, sys_path)
+        stub = try_to_load_stub_cached(inference_state, import_names, python_value_set,
+                                       parent_module_value, sys_path)
         if stub is not None:
             return ValueSet([stub])
         return python_value_set
@@ -128,7 +130,10 @@ def import_module_decorator(func):
     return wrapper
 
 
-def _try_to_load_stub_cached(inference_state, import_names, *args, **kwargs):
+def try_to_load_stub_cached(inference_state, import_names, *args, **kwargs):
+    if import_names is None:
+        return None
+
     try:
         return inference_state.stub_module_cache[import_names]
     except KeyError:
@@ -152,7 +157,7 @@ def _try_to_load_stub(inference_state, import_names, python_value_set,
     """
     if parent_module_value is None and len(import_names) > 1:
         try:
-            parent_module_value = _try_to_load_stub_cached(
+            parent_module_value = try_to_load_stub_cached(
                 inference_state, import_names[:-1], NO_VALUES,
                 parent_module_value=None, sys_path=sys_path)
         except KeyError:
@@ -162,7 +167,6 @@ def _try_to_load_stub(inference_state, import_names, python_value_set,
     if len(import_names) == 1:
         # foo-stubs
         for p in sys_path:
-            p = cast_path(p)
             init = os.path.join(p, *import_names) + '-stubs' + os.path.sep + '__init__.pyi'
             m = _try_to_load_stub_from_file(
                 inference_state,
@@ -172,6 +176,13 @@ def _try_to_load_stub(inference_state, import_names, python_value_set,
             )
             if m is not None:
                 return m
+        if import_names[0] == 'django' and python_value_set:
+            return _try_to_load_stub_from_file(
+                inference_state,
+                python_value_set,
+                file_io=FileIO(str(DJANGO_INIT_PATH)),
+                import_names=import_names,
+            )
 
     # 2. Try to load pyi files next to py files.
     for c in python_value_set:
@@ -184,8 +195,8 @@ def _try_to_load_stub(inference_state, import_names, python_value_set,
             file_paths = []
             if c.is_namespace():
                 file_paths = [os.path.join(p, '__init__.pyi') for p in c.py__path__()]
-            elif file_path is not None and file_path.endswith('.py'):
-                file_paths = [file_path + 'i']
+            elif file_path is not None and file_path.suffix == '.py':
+                file_paths = [str(file_path) + 'i']
 
             for file_path in file_paths:
                 m = _try_to_load_stub_from_file(
@@ -239,38 +250,49 @@ def _load_from_typeshed(inference_state, python_value_set, parent_module_value,
             # Only if it's a package (= a folder) something can be
             # imported.
             return None
-        path = parent_module_value.py__path__()
-        map_ = _merge_create_stub_map(path)
+        paths = parent_module_value.py__path__()
+        # Once the initial package has been loaded, the sub packages will
+        # always be loaded, regardless if they are there or not. This makes
+        # sense, IMO, because stubs take preference, even if the original
+        # library doesn't provide a module (it could be dynamic). ~dave
+        map_ = _merge_create_stub_map([PathInfo(p, is_third_party=False) for p in paths])
 
     if map_ is not None:
-        path = map_.get(import_name)
-        if path is not None:
+        path_info = map_.get(import_name)
+        if path_info is not None and (not path_info.is_third_party or python_value_set):
             return _try_to_load_stub_from_file(
                 inference_state,
                 python_value_set,
-                file_io=FileIO(path),
+                file_io=FileIO(path_info.path),
                 import_names=import_names,
             )
 
 
 def _try_to_load_stub_from_file(inference_state, python_value_set, file_io, import_names):
     try:
-        stub_module_node = inference_state.parse(
-            file_io=file_io,
-            cache=True,
-            use_latest_grammar=True
-        )
-    except (OSError, IOError):  # IOError is Python 2 only
+        stub_module_node = parse_stub_module(inference_state, file_io)
+    except OSError:
         # The file that you're looking for doesn't exist (anymore).
         return None
     else:
         return create_stub_module(
-            inference_state, python_value_set, stub_module_node, file_io,
-            import_names
+            inference_state, inference_state.latest_grammar, python_value_set,
+            stub_module_node, file_io, import_names
         )
 
 
-def create_stub_module(inference_state, python_value_set, stub_module_node, file_io, import_names):
+def parse_stub_module(inference_state, file_io):
+    return inference_state.parse(
+        file_io=file_io,
+        cache=True,
+        diff_cache=settings.fast_parser,
+        cache_path=settings.cache_directory,
+        use_latest_grammar=True
+    )
+
+
+def create_stub_module(inference_state, grammar, python_value_set,
+                       stub_module_node, file_io, import_names):
     if import_names == ('typing',):
         module_cls = TypingModuleWrapper
     else:
@@ -282,7 +304,7 @@ def create_stub_module(inference_state, python_value_set, stub_module_node, file
         string_names=import_names,
         # The code was loaded with latest_grammar, so use
         # that.
-        code_lines=get_cached_code_lines(inference_state.latest_grammar, file_io.path),
+        code_lines=get_cached_code_lines(grammar, file_io.path),
         is_package=file_name == '__init__.pyi',
     )
     return stub_module_value

jedi/inference/gradual/typing.py

@@ -5,16 +5,19 @@ values.
 
 This file deals with all the typing.py cases.
 """
+import itertools
+
 from jedi import debug
-from jedi.inference.compiled import builtin_from_name
+from jedi.inference.compiled import builtin_from_name, create_simple_object
 from jedi.inference.base_value import ValueSet, NO_VALUES, Value, \
-    LazyValueWrapper
+    LazyValueWrapper, ValueWrapper
 from jedi.inference.lazy_value import LazyKnownValues
 from jedi.inference.arguments import repack_with_argument_clinic
 from jedi.inference.filters import FilterWrapper
 from jedi.inference.names import NameWrapper, ValueName
 from jedi.inference.value.klass import ClassMixin
-from jedi.inference.gradual.base import BaseTypingValue, BaseTypingValueWithGenerics
+from jedi.inference.gradual.base import BaseTypingValue, \
+    BaseTypingClassWithGenerics, BaseTypingInstance
 from jedi.inference.gradual.type_var import TypeVarClass
 from jedi.inference.gradual.generics import LazyGenericManager, TupleGenericManager
 
@@ -29,7 +32,7 @@ _TYPE_ALIAS_TYPES = {
     'DefaultDict': 'collections.defaultdict',
     'Deque': 'collections.deque',
 }
-_PROXY_TYPES = 'Optional Union ClassVar'.split()
+_PROXY_TYPES = 'Optional Union ClassVar Annotated'.split()
 
 
 class TypingModuleName(NameWrapper):
@@ -60,43 +63,41 @@ class TypingModuleName(NameWrapper):
             # have any effects there (because it's never executed).
             return
         elif name == 'TypeVar':
-            yield TypeVarClass.create_cached(
-                inference_state, self.parent_context, self.tree_name)
+            cls, = self._wrapped_name.infer()
+            yield TypeVarClass.create_cached(inference_state, cls)
         elif name == 'Any':
-            yield Any.create_cached(
+            yield AnyClass.create_cached(
                 inference_state, self.parent_context, self.tree_name)
         elif name == 'TYPE_CHECKING':
             # This is needed for e.g. imports that are only available for type
             # checking or are in cycles. The user can then check this variable.
-            yield builtin_from_name(inference_state, u'True')
+            yield builtin_from_name(inference_state, 'True')
         elif name == 'overload':
             yield OverloadFunction.create_cached(
                 inference_state, self.parent_context, self.tree_name)
         elif name == 'NewType':
-            yield NewTypeFunction.create_cached(
-                inference_state, self.parent_context, self.tree_name)
+            v, = self._wrapped_name.infer()
+            yield NewTypeFunction.create_cached(inference_state, v)
         elif name == 'cast':
-            yield CastFunction.create_cached(
-                inference_state, self.parent_context, self.tree_name)
+            cast_fn, = self._wrapped_name.infer()
+            yield CastFunction.create_cached(inference_state, cast_fn)
         elif name == 'TypedDict':
             # TODO doesn't even exist in typeshed/typing.py, yet. But will be
             # added soon.
-            pass
-        elif name in ('no_type_check', 'no_type_check_decorator'):
-            # This is not necessary, as long as we are not doing type checking.
-            for c in self._wrapped_name.infer():  # Fuck my life Python 2
-                yield c
+            yield TypedDictClass.create_cached(
+                inference_state, self.parent_context, self.tree_name)
         else:
-            # Everything else shouldn't be relevant for type checking.
-            for c in self._wrapped_name.infer():  # Fuck my life Python 2
-                yield c
+            # Not necessary, as long as we are not doing type checking:
+            # no_type_check & no_type_check_decorator
+            # Everything else shouldn't be relevant...
+            yield from self._wrapped_name.infer()
 
 
 class TypingModuleFilterWrapper(FilterWrapper):
     name_wrapper_class = TypingModuleName
 
 
-class TypingValueWithIndex(BaseTypingValueWithGenerics):
+class ProxyWithGenerics(BaseTypingClassWithGenerics):
     def execute_annotation(self):
         string_name = self._tree_name.value
 
@@ -108,11 +109,11 @@ class TypingValueWithIndex(BaseTypingValueWithGenerics):
             # Optional is basically just saying it's either None or the actual
             # type.
             return self.gather_annotation_classes().execute_annotation() \
-                | ValueSet([builtin_from_name(self.inference_state, u'None')])
+                | ValueSet([builtin_from_name(self.inference_state, 'None')])
         elif string_name == 'Type':
             # The type is actually already given in the index_value
             return self._generics_manager[0]
-        elif string_name == 'ClassVar':
+        elif string_name in ['ClassVar', 'Annotated']:
             # For now don't do anything here, ClassVars are always used.
             return self._generics_manager[0].execute_annotation()
 
@@ -125,6 +126,7 @@ class TypingValueWithIndex(BaseTypingValueWithGenerics):
         cls = mapped[string_name]
         return ValueSet([cls(
             self.parent_context,
+            self,
             self._tree_name,
             generics_manager=self._generics_manager,
         )])
@@ -133,15 +135,33 @@ class TypingValueWithIndex(BaseTypingValueWithGenerics):
         return ValueSet.from_sets(self._generics_manager.to_tuple())
 
     def _create_instance_with_generics(self, generics_manager):
-        return TypingValueWithIndex(
+        return ProxyWithGenerics(
             self.parent_context,
             self._tree_name,
             generics_manager
         )
 
+    def infer_type_vars(self, value_set):
+        annotation_generics = self.get_generics()
+
+        if not annotation_generics:
+            return {}
+
+        annotation_name = self.py__name__()
+        if annotation_name == 'Optional':
+            # Optional[T] is equivalent to Union[T, None]. In Jedi unions
+            # are represented by members within a ValueSet, so we extract
+            # the T from the Optional[T] by removing the None value.
+            none = builtin_from_name(self.inference_state, 'None')
+            return annotation_generics[0].infer_type_vars(
+                value_set.filter(lambda x: x != none),
+            )
+
+        return {}
+
 
 class ProxyTypingValue(BaseTypingValue):
-    index_class = TypingValueWithIndex
+    index_class = ProxyWithGenerics
 
     def with_generics(self, generics_tuple):
         return self.index_class.create_cached(
@@ -179,12 +199,45 @@ class _TypingClassMixin(ClassMixin):
         return ValueName(self, self._tree_name)
 
 
-class TypingClassValueWithIndex(_TypingClassMixin, TypingValueWithIndex):
-    pass
+class TypingClassWithGenerics(ProxyWithGenerics, _TypingClassMixin):
+    def infer_type_vars(self, value_set):
+        type_var_dict = {}
+        annotation_generics = self.get_generics()
+
+        if not annotation_generics:
+            return type_var_dict
+
+        annotation_name = self.py__name__()
+        if annotation_name == 'Type':
+            return annotation_generics[0].infer_type_vars(
+                # This is basically a trick to avoid extra code: We execute the
+                # incoming classes to be able to use the normal code for type
+                # var inference.
+                value_set.execute_annotation(),
+            )
+
+        elif annotation_name == 'Callable':
+            if len(annotation_generics) == 2:
+                return annotation_generics[1].infer_type_vars(
+                    value_set.execute_annotation(),
+                )
+
+        elif annotation_name == 'Tuple':
+            tuple_annotation, = self.execute_annotation()
+            return tuple_annotation.infer_type_vars(value_set)
+
+        return type_var_dict
+
+    def _create_instance_with_generics(self, generics_manager):
+        return TypingClassWithGenerics(
+            self.parent_context,
+            self._tree_name,
+            generics_manager
+        )
 
 
-class ProxyTypingClassValue(_TypingClassMixin, ProxyTypingValue):
-    index_class = TypingClassValueWithIndex
+class ProxyTypingClassValue(ProxyTypingValue, _TypingClassMixin):
+    index_class = TypingClassWithGenerics
 
 
 class TypeAlias(LazyValueWrapper):
@@ -206,8 +259,6 @@ class TypeAlias(LazyValueWrapper):
 
     def _get_wrapped_value(self):
         module_name, class_name = self._actual.split('.')
-        if self.inference_state.environment.version_info.major == 2 and module_name == 'builtins':
-            module_name = '__builtin__'
 
         # TODO use inference_state.import_module?
         from jedi.inference.imports import Importer
@@ -223,8 +274,11 @@ class TypeAlias(LazyValueWrapper):
     def gather_annotation_classes(self):
         return ValueSet([self._get_wrapped_value()])
 
+    def get_signatures(self):
+        return []
 
-class Callable(BaseTypingValueWithGenerics):
+
+class Callable(BaseTypingInstance):
     def py__call__(self, arguments):
         """
             def x() -> Callable[[Callable[..., _T]], _T]: ...
@@ -240,13 +294,11 @@ class Callable(BaseTypingValueWithGenerics):
             from jedi.inference.gradual.annotation import infer_return_for_callable
             return infer_return_for_callable(arguments, param_values, result_values)
 
+    def py__get__(self, instance, class_value):
+        return ValueSet([self])
 
-class Tuple(LazyValueWrapper):
-    def __init__(self, parent_context, name, generics_manager):
-        self.inference_state = parent_context.inference_state
-        self.parent_context = parent_context
-        self._generics_manager = generics_manager
 
+class Tuple(BaseTypingInstance):
     def _is_homogenous(self):
         # To specify a variable-length tuple of homogeneous type, Tuple[T, ...]
         # is used.
@@ -282,16 +334,60 @@ class Tuple(LazyValueWrapper):
             .py__getattribute__('tuple').execute_annotation()
         return tuple_
 
+    @property
+    def name(self):
+        return self._wrapped_value.name
 
-class Generic(BaseTypingValueWithGenerics):
+    def infer_type_vars(self, value_set):
+        # Circular
+        from jedi.inference.gradual.annotation import merge_pairwise_generics, merge_type_var_dicts
+
+        value_set = value_set.filter(
+            lambda x: x.py__name__().lower() == 'tuple',
+        )
+
+        if self._is_homogenous():
+            # The parameter annotation is of the form `Tuple[T, ...]`,
+            # so we treat the incoming tuple like a iterable sequence
+            # rather than a positional container of elements.
+            return self._class_value.get_generics()[0].infer_type_vars(
+                value_set.merge_types_of_iterate(),
+            )
+
+        else:
+            # The parameter annotation has only explicit type parameters
+            # (e.g: `Tuple[T]`, `Tuple[T, U]`, `Tuple[T, U, V]`, etc.) so we
+            # treat the incoming values as needing to match the annotation
+            # exactly, just as we would for non-tuple annotations.
+
+            type_var_dict = {}
+            for element in value_set:
+                try:
+                    method = element.get_annotated_class_object
+                except AttributeError:
+                    # This might still happen, because the tuple name matching
+                    # above is not 100% correct, so just catch the remaining
+                    # cases here.
+                    continue
+
+                py_class = method()
+                merge_type_var_dicts(
+                    type_var_dict,
+                    merge_pairwise_generics(self._class_value, py_class),
+                )
+
+            return type_var_dict
+
+
+class Generic(BaseTypingInstance):
     pass
 
 
-class Protocol(BaseTypingValueWithGenerics):
+class Protocol(BaseTypingInstance):
     pass
 
 
-class Any(BaseTypingValue):
+class AnyClass(BaseTypingValue):
     def execute_annotation(self):
         debug.warning('Used Any - returned no results')
         return NO_VALUES
@@ -304,7 +400,7 @@ class OverloadFunction(BaseTypingValue):
         return func_value_set
 
 
-class NewTypeFunction(BaseTypingValue):
+class NewTypeFunction(ValueWrapper):
     def py__call__(self, arguments):
         ordered_args = arguments.unpack()
         next(ordered_args, (None, None))
@@ -322,10 +418,14 @@ class NewTypeFunction(BaseTypingValue):
 
 class NewType(Value):
     def __init__(self, inference_state, parent_context, tree_node, type_value_set):
-        super(NewType, self).__init__(inference_state, parent_context)
+        super().__init__(inference_state, parent_context)
         self._type_value_set = type_value_set
         self.tree_node = tree_node
 
+    def py__class__(self):
+        c, = self._type_value_set.py__class__()
+        return c
+
     def py__call__(self, arguments):
         return self._type_value_set.execute_annotation()
 
@@ -334,8 +434,55 @@ class NewType(Value):
         from jedi.inference.compiled.value import CompiledValueName
         return CompiledValueName(self, 'NewType')
 
+    def __repr__(self) -> str:
+        return '<NewType: %s>%s' % (self.tree_node, self._type_value_set)
 
-class CastFunction(BaseTypingValue):
+
+class CastFunction(ValueWrapper):
     @repack_with_argument_clinic('type, object, /')
     def py__call__(self, type_value_set, object_value_set):
         return type_value_set.execute_annotation()
+
+
+class TypedDictClass(BaseTypingValue):
+    """
+    This class has no responsibilities and is just here to make sure that typed
+    dicts can be identified.
+    """
+
+
+class TypedDict(LazyValueWrapper):
+    """Represents the instance version of ``TypedDictClass``."""
+    def __init__(self, definition_class):
+        self.inference_state = definition_class.inference_state
+        self.parent_context = definition_class.parent_context
+        self.tree_node = definition_class.tree_node
+        self._definition_class = definition_class
+
+    @property
+    def name(self):
+        return ValueName(self, self.tree_node.name)
+
+    def py__simple_getitem__(self, index):
+        if isinstance(index, str):
+            return ValueSet.from_sets(
+                name.infer()
+                for filter in self._definition_class.get_filters(is_instance=True)
+                for name in filter.get(index)
+            )
+        return NO_VALUES
+
+    def get_key_values(self):
+        filtered_values = itertools.chain.from_iterable((
+            f.values()
+            for f in self._definition_class.get_filters(is_instance=True)
+        ))
+        return ValueSet({
+            create_simple_object(self.inference_state, v.string_name)
+            for v in filtered_values
+        })
+
+    def _get_wrapped_value(self):
+        d, = self.inference_state.builtins_module.py__getattribute__('dict')
+        result, = d.execute_with_values()
+        return result

jedi/inference/gradual/utils.py

@@ -1,29 +1,34 @@
-import os
+from pathlib import Path
 
 from jedi.inference.gradual.typeshed import TYPESHED_PATH, create_stub_module
 
 
-def load_proper_stub_module(inference_state, file_io, import_names, module_node):
+def load_proper_stub_module(inference_state, grammar, file_io, import_names, module_node):
     """
     This function is given a random .pyi file and should return the proper
     module.
     """
     path = file_io.path
-    assert path.endswith('.pyi')
-    if path.startswith(TYPESHED_PATH):
-        # /foo/stdlib/3/os/__init__.pyi -> stdlib/3/os/__init__
-        rest = path[len(TYPESHED_PATH) + 1: -4]
-        split_paths = tuple(rest.split(os.path.sep))
-        # Remove the stdlib/3 or third_party/3.5 part
-        import_names = split_paths[2:]
-        if import_names[-1] == '__init__':
+    path = Path(path)
+    assert path.suffix == '.pyi'
+    try:
+        relative_path = path.relative_to(TYPESHED_PATH)
+    except ValueError:
+        pass
+    else:
+        # /[...]/stdlib/3/os/__init__.pyi -> stdlib/3/os/__init__
+        rest = relative_path.with_suffix('')
+        # Remove the stdlib/3 or third_party/3.6 part
+        import_names = rest.parts[2:]
+        if rest.name == '__init__':
             import_names = import_names[:-1]
 
     if import_names is not None:
         actual_value_set = inference_state.import_module(import_names, prefer_stubs=False)
 
         stub = create_stub_module(
-            inference_state, actual_value_set, module_node, file_io, import_names
+            inference_state, grammar, actual_value_set,
+            module_node, file_io, import_names
         )
         inference_state.stub_module_cache[import_names] = stub
         return stub

jedi/inference/helpers.py

@@ -7,19 +7,17 @@ from contextlib import contextmanager
 
 from parso.python import tree
 
-from jedi._compatibility import unicode
-from jedi.parser_utils import get_parent_scope
-
 
 def is_stdlib_path(path):
     # Python standard library paths look like this:
-    # /usr/lib/python3.5/...
+    # /usr/lib/python3.9/...
     # TODO The implementation below is probably incorrect and not complete.
-    if 'dist-packages' in path or 'site-packages' in path:
+    parts = path.parts
+    if 'dist-packages' in parts or 'site-packages' in parts:
         return False
 
     base_path = os.path.join(sys.prefix, 'lib', 'python')
-    return bool(re.match(re.escape(base_path) + r'\d.\d', path))
+    return bool(re.match(re.escape(base_path) + r'\d.\d', str(path)))
 
 
 def deep_ast_copy(obj):
@@ -122,35 +120,8 @@ def get_names_of_node(node):
         return list(chain.from_iterable(get_names_of_node(c) for c in children))
 
 
-def get_module_names(module, all_scopes):
-    """
-    Returns a dictionary with name parts as keys and their call paths as
-    values.
-    """
-    names = list(chain.from_iterable(module.get_used_names().values()))
-    if not all_scopes:
-        # We have to filter all the names that don't have the module as a
-        # parent_scope. There's None as a parent, because nodes in the module
-        # node have the parent module and not suite as all the others.
-        # Therefore it's important to catch that case.
-
-        def is_module_scope_name(name):
-            parent_scope = get_parent_scope(name)
-            # async functions have an extra wrapper. Strip it.
-            if parent_scope and parent_scope.type == 'async_stmt':
-                parent_scope = parent_scope.parent
-            return parent_scope in (module, None)
-
-        names = [n for n in names if is_module_scope_name(n)]
-    return names
-
-
 def is_string(value):
-    if value.inference_state.environment.version_info.major == 2:
-        str_classes = (unicode, bytes)
-    else:
-        str_classes = (unicode,)
-    return value.is_compiled() and isinstance(value.get_safe_value(default=None), str_classes)
+    return value.is_compiled() and isinstance(value.get_safe_value(default=None), str)
 
 
 def is_literal(value):
@@ -168,7 +139,7 @@ def get_int_or_none(value):
 
 
 def get_str_or_none(value):
-    return _get_safe_value_or_none(value, (bytes, unicode))
+    return _get_safe_value_or_none(value, str)
 
 
 def is_number(value):

jedi/inference/imports.py

@@ -5,20 +5,18 @@ not any actual importing done. This module is about finding modules in the
 filesystem. This can be quite tricky sometimes, because Python imports are not
 always that simple.
 
-This module uses imp for python up to 3.2 and importlib for python 3.3 on; the
-correct implementation is delegated to _compatibility.
-
 This module also supports import autocompletion, which means to complete
 statements like ``from datetim`` (cursor at the end would return ``datetime``).
 """
 import os
+from pathlib import Path
 
 from parso.python import tree
 from parso.tree import search_ancestor
 
-from jedi._compatibility import ImplicitNSInfo, force_unicode
 from jedi import debug
 from jedi import settings
+from jedi.file_io import FolderIO
 from jedi.parser_utils import get_cached_code_lines
 from jedi.inference import sys_path
 from jedi.inference import helpers
@@ -28,12 +26,13 @@ from jedi.inference.utils import unite
 from jedi.inference.cache import inference_state_method_cache
 from jedi.inference.names import ImportName, SubModuleName
 from jedi.inference.base_value import ValueSet, NO_VALUES
-from jedi.inference.gradual.typeshed import import_module_decorator
-from jedi.inference.value.module import iter_module_names
+from jedi.inference.gradual.typeshed import import_module_decorator, \
+    create_stub_module, parse_stub_module
+from jedi.inference.compiled.subprocess.functions import ImplicitNSInfo
 from jedi.plugins import plugin_manager
 
 
-class ModuleCache(object):
+class ModuleCache:
     def __init__(self):
         self._name_cache = {}
 
@@ -151,7 +150,7 @@ def _level_to_base_import_path(project_path, directory, level):
             return None, directory
 
 
-class Importer(object):
+class Importer:
     def __init__(self, inference_state, import_path, module_context, level=0):
         """
         An implementation similar to ``__import__``. Use `follow`
@@ -191,25 +190,26 @@ class Importer(object):
                 import_path = base + tuple(import_path)
             else:
                 path = module_context.py__file__()
+                project_path = self._inference_state.project.path
                 import_path = list(import_path)
                 if path is None:
                     # If no path is defined, our best guess is that the current
                     # file is edited by a user on the current working
                     # directory. We need to add an initial path, because it
                     # will get removed as the name of the current file.
-                    directory = os.getcwd()
+                    directory = project_path
                 else:
                     directory = os.path.dirname(path)
 
                 base_import_path, base_directory = _level_to_base_import_path(
-                    self._inference_state.project._path, directory, level,
+                    project_path, directory, level,
                 )
                 if base_directory is None:
                     # Everything is lost, the relative import does point
                     # somewhere out of the filesystem.
                     self._infer_possible = False
                 else:
-                    self._fixed_sys_path = [force_unicode(base_directory)]
+                    self._fixed_sys_path = [base_directory]
 
                 if base_import_path is None:
                     if import_path:
@@ -238,11 +238,49 @@ class Importer(object):
             # inference we want to show the user as much as possible.
             # See GH #1446.
             self._inference_state.get_sys_path(add_init_paths=not is_completion)
-            + sys_path.check_sys_path_modifications(self._module_context)
+            + [
+                str(p) for p
+                in sys_path.check_sys_path_modifications(self._module_context)
+            ]
         )
 
     def follow(self):
-        if not self.import_path or not self._infer_possible:
+        if not self.import_path:
+            if self._fixed_sys_path:
+                # This is a bit of a special case, that maybe should be
+                # revisited. If the project path is wrong or the user uses
+                # relative imports the wrong way, we might end up here, where
+                # the `fixed_sys_path == project.path` in that case we kind of
+                # use the project.path.parent directory as our path. This is
+                # usually not a problem, except if imports in other places are
+                # using the same names. Example:
+                #
+                # foo/                       < #1
+                #   - setup.py
+                #   - foo/                   < #2
+                #     - __init__.py
+                #     - foo.py               < #3
+                #
+                # If the top foo is our project folder and somebody uses
+                # `from . import foo` in `setup.py`, it will resolve to foo #2,
+                # which means that the import for foo.foo is cached as
+                # `__init__.py` (#2) and not as `foo.py` (#3). This is usually
+                # not an issue, because this case is probably pretty rare, but
+                # might be an issue for some people.
+                #
+                # However for most normal cases where we work with different
+                # file names, this code path hits where we basically change the
+                # project path to an ancestor of project path.
+                from jedi.inference.value.namespace import ImplicitNamespaceValue
+                import_path = (os.path.basename(self._fixed_sys_path[0]),)
+                ns = ImplicitNamespaceValue(
+                    self._inference_state,
+                    string_names=import_path,
+                    paths=self._fixed_sys_path,
+                )
+                return ValueSet({ns})
+            return NO_VALUES
+        if not self._infer_possible:
             return NO_VALUES
 
         # Check caches first
@@ -264,24 +302,15 @@ class Importer(object):
         Get the names of all modules in the search_path. This means file names
         and not names defined in the files.
         """
-        names = []
-        # add builtin module names
-        if search_path is None and in_module is None:
-            names += [
-                ImportName(self._module_context, name)
-                for name in self._inference_state.compiled_subprocess.get_builtin_module_names()
-            ]
-
         if search_path is None:
-            search_path = self._sys_path_with_modifications(is_completion=True)
-
-        for name in iter_module_names(self._inference_state, search_path):
-            if in_module is None:
-                n = ImportName(self._module_context, name)
-            else:
-                n = SubModuleName(in_module.as_context(), name)
-            names.append(n)
-        return names
+            sys_path = self._sys_path_with_modifications(is_completion=True)
+        else:
+            sys_path = search_path
+        return list(iter_module_names(
+            self._inference_state, self._module_context, sys_path,
+            module_cls=ImportName if in_module is None else SubModuleName,
+            add_builtin_modules=search_path is None and in_module is None,
+        ))
 
     def completion_names(self, inference_state, only_modules=False):
         """
@@ -310,7 +339,7 @@ class Importer(object):
             values = self.follow()
             for value in values:
                 # Non-modules are not completable.
-                if value.api_type != 'module':  # not a module
+                if value.api_type not in ('module', 'namespace'):  # not a module
                     continue
                 if not value.is_compiled():
                     # sub_modules_dict is not implemented for compiled modules.
@@ -339,7 +368,7 @@ def import_module_by_names(inference_state, import_names, sys_path=None,
         sys_path = inference_state.get_sys_path()
 
     str_import_names = tuple(
-        force_unicode(i.value if isinstance(i, tree.Name) else i)
+        i.value if isinstance(i, tree.Name) else i
         for i in import_names
     )
     value_set = [None]
@@ -393,20 +422,13 @@ def import_module(inference_state, import_names, parent_module_value, sys_path):
             # The module might not be a package.
             return NO_VALUES
 
-        for path in paths:
-            # At the moment we are only using one path. So this is
-            # not important to be correct.
-            if not isinstance(path, list):
-                path = [path]
-            file_io_or_ns, is_pkg = inference_state.compiled_subprocess.get_module_info(
-                string=import_names[-1],
-                path=path,
-                full_name=module_name,
-                is_global_search=False,
-            )
-            if is_pkg is not None:
-                break
-        else:
+        file_io_or_ns, is_pkg = inference_state.compiled_subprocess.get_module_info(
+            string=import_names[-1],
+            path=paths,
+            full_name=module_name,
+            is_global_search=False,
+        )
+        if is_pkg is None:
             return NO_VALUES
 
     if isinstance(file_io_or_ns, ImplicitNSInfo):
@@ -440,7 +462,7 @@ def _load_python_module(inference_state, file_io,
         file_io=file_io,
         cache=True,
         diff_cache=settings.fast_parser,
-        cache_path=settings.cache_directory
+        cache_path=settings.cache_directory,
     )
 
     from jedi.inference.value import ModuleValue
@@ -454,8 +476,12 @@ def _load_python_module(inference_state, file_io,
 
 
 def _load_builtin_module(inference_state, import_names=None, sys_path=None):
+    project = inference_state.project
     if sys_path is None:
         sys_path = inference_state.get_sys_path()
+    if not project._load_unsafe_extensions:
+        safe_paths = project._get_base_sys_path(inference_state)
+        sys_path = [p for p in sys_path if p in safe_paths]
 
     dotted_name = '.'.join(import_names)
     assert dotted_name is not None
@@ -467,32 +493,59 @@ def _load_builtin_module(inference_state, import_names=None, sys_path=None):
     return module
 
 
-def load_module_from_path(inference_state, file_io, base_names=None):
+def load_module_from_path(inference_state, file_io, import_names=None, is_package=None):
     """
     This should pretty much only be used for get_modules_containing_name. It's
     here to ensure that a random path is still properly loaded into the Jedi
     module structure.
     """
-    path = file_io.path
-    if base_names:
-        module_name = os.path.basename(path)
-        module_name = sys_path.remove_python_path_suffix(module_name)
-        is_package = module_name == '__init__'
-        if is_package:
-            import_names = base_names
-        else:
-            import_names = base_names + (module_name,)
-    else:
+    path = Path(file_io.path)
+    if import_names is None:
         e_sys_path = inference_state.get_sys_path()
         import_names, is_package = sys_path.transform_path_to_dotted(e_sys_path, path)
+    else:
+        assert isinstance(is_package, bool)
 
-    module = _load_python_module(
-        inference_state, file_io,
-        import_names=import_names,
-        is_package=is_package,
+    is_stub = path.suffix == '.pyi'
+    if is_stub:
+        folder_io = file_io.get_parent_folder()
+        if folder_io.path.endswith('-stubs'):
+            folder_io = FolderIO(folder_io.path[:-6])
+        if path.name == '__init__.pyi':
+            python_file_io = folder_io.get_file_io('__init__.py')
+        else:
+            python_file_io = folder_io.get_file_io(import_names[-1] + '.py')
+
+        try:
+            v = load_module_from_path(
+                inference_state, python_file_io,
+                import_names, is_package=is_package
+            )
+            values = ValueSet([v])
+        except FileNotFoundError:
+            values = NO_VALUES
+
+        return create_stub_module(
+            inference_state, inference_state.latest_grammar, values,
+            parse_stub_module(inference_state, file_io), file_io, import_names
+        )
+    else:
+        module = _load_python_module(
+            inference_state, file_io,
+            import_names=import_names,
+            is_package=is_package,
+        )
+        inference_state.module_cache.add(import_names, ValueSet([module]))
+        return module
+
+
+def load_namespace_from_path(inference_state, folder_io):
+    import_names, is_package = sys_path.transform_path_to_dotted(
+        inference_state.get_sys_path(),
+        Path(folder_io.path)
     )
-    inference_state.module_cache.add(import_names, ValueSet([module]))
-    return module
+    from jedi.inference.value.namespace import ImplicitNamespaceValue
+    return ImplicitNamespaceValue(inference_state, import_names, [folder_io.path])
 
 
 def follow_error_node_imports_if_possible(context, name):
@@ -522,3 +575,18 @@ def follow_error_node_imports_if_possible(context, name):
             return Importer(
                 context.inference_state, names, context.get_root_context(), level).follow()
     return None
+
+
+def iter_module_names(inference_state, module_context, search_path,
+                      module_cls=ImportName, add_builtin_modules=True):
+    """
+    Get the names of all modules in the search_path. This means file names
+    and not names defined in the files.
+    """
+    # add builtin module names
+    if add_builtin_modules:
+        for name in inference_state.compiled_subprocess.get_builtin_module_names():
+            yield module_cls(module_context, name)
+
+    for name in inference_state.compiled_subprocess.iter_module_names(search_path):
+        yield module_cls(module_context, name)

jedi/inference/lazy_value.py

@@ -1,10 +1,12 @@
 from jedi.inference.base_value import ValueSet, NO_VALUES
-from jedi.common.utils import monkeypatch
+from jedi.common import monkeypatch
 
 
-class AbstractLazyValue(object):
-    def __init__(self, data):
+class AbstractLazyValue:
+    def __init__(self, data, min=1, max=1):
         self.data = data
+        self.min = min
+        self.max = max
 
     def __repr__(self):
         return '<%s: %s>' % (self.__class__.__name__, self.data)
@@ -26,16 +28,16 @@ class LazyKnownValues(AbstractLazyValue):
 
 
 class LazyUnknownValue(AbstractLazyValue):
-    def __init__(self):
-        super(LazyUnknownValue, self).__init__(None)
+    def __init__(self, min=1, max=1):
+        super().__init__(None, min, max)
 
     def infer(self):
         return NO_VALUES
 
 
 class LazyTreeValue(AbstractLazyValue):
-    def __init__(self, context, node):
-        super(LazyTreeValue, self).__init__(node)
+    def __init__(self, context, node, min=1, max=1):
+        super().__init__(node, min, max)
         self.context = context
         # We need to save the predefined names. It's an unfortunate side effect
         # that needs to be tracked otherwise results will be wrong.

jedi/inference/names.py

@@ -1,11 +1,13 @@
 from abc import abstractmethod
+from inspect import Parameter
+from typing import Optional, Tuple
 
 from parso.tree import search_ancestor
 
-from jedi._compatibility import Parameter
 from jedi.parser_utils import find_statement_documentation, clean_scope_docstring
 from jedi.inference.utils import unite
 from jedi.inference.base_value import ValueSet, NO_VALUES
+from jedi.inference.cache import inference_state_method_cache
 from jedi.inference import docstrings
 from jedi.cache import memoize_method
 from jedi.inference.helpers import deep_ast_copy, infer_call_of_leaf
@@ -23,9 +25,9 @@ def _merge_name_docs(names):
     return doc
 
 
-class AbstractNameDefinition(object):
-    start_pos = None
-    string_name = None
+class AbstractNameDefinition:
+    start_pos: Optional[Tuple[int, int]] = None
+    string_name: str
     parent_context = None
     tree_name = None
     is_value_name = True
@@ -123,7 +125,7 @@ class AbstractTreeName(AbstractNameDefinition):
             else:
                 return None
 
-        return super(AbstractTreeName, self).get_qualified_names(include_module_names)
+        return super().get_qualified_names(include_module_names)
 
     def _get_qualified_names(self):
         parent_names = self.parent_context.get_qualified_names()
@@ -223,7 +225,7 @@ class AbstractTreeName(AbstractNameDefinition):
         return self.tree_name.start_pos
 
 
-class ValueNameMixin(object):
+class ValueNameMixin:
     def infer(self):
         return ValueSet([self._value])
 
@@ -242,11 +244,11 @@ class ValueNameMixin(object):
     def get_root_context(self):
         if self.parent_context is None:  # A module
             return self._value.as_context()
-        return super(ValueNameMixin, self).get_root_context()
+        return super().get_root_context()
 
     def get_defining_qualified_value(self):
         context = self.parent_context
-        if context.is_module() or context.is_class():
+        if context is not None and (context.is_module() or context.is_class()):
             return self.parent_context.get_value()  # Might be None
         return None
 
@@ -257,7 +259,7 @@ class ValueNameMixin(object):
 
 class ValueName(ValueNameMixin, AbstractTreeName):
     def __init__(self, value, tree_name):
-        super(ValueName, self).__init__(value.parent_context, tree_name)
+        super().__init__(value.parent_context, tree_name)
         self._value = value
 
     def goto(self):
@@ -330,9 +332,21 @@ class TreeNameDefinition(AbstractTreeName):
             node = node.parent
         return indexes
 
+    @property
+    def inference_state(self):
+        # Used by the cache function below
+        return self.parent_context.inference_state
+
+    @inference_state_method_cache(default='')
     def py__doc__(self):
         api_type = self.api_type
-        if api_type in ('function', 'class'):
+        if api_type in ('function', 'class', 'property'):
+            if self.parent_context.get_root_context().is_stub():
+                from jedi.inference.gradual.conversion import convert_names
+                names = convert_names([self], prefer_stub_to_compiled=False)
+                if self not in names:
+                    return _merge_name_docs(names)
+
             # Make sure the names are not TreeNameDefinitions anymore.
             return clean_scope_docstring(self.tree_name.get_definition())
 
@@ -346,7 +360,7 @@ class TreeNameDefinition(AbstractTreeName):
         return ''
 
 
-class _ParamMixin(object):
+class _ParamMixin:
     def maybe_positional_argument(self, include_star=True):
         options = [Parameter.POSITIONAL_ONLY, Parameter.POSITIONAL_OR_KEYWORD]
         if include_star:
@@ -372,7 +386,7 @@ class _ParamMixin(object):
 
 
 class ParamNameInterface(_ParamMixin):
-    api_type = u'param'
+    api_type = 'param'
 
     def get_kind(self):
         raise NotImplementedError
@@ -400,6 +414,9 @@ class ParamNameInterface(_ParamMixin):
             return 2
         return 0
 
+    def infer_default(self):
+        return NO_VALUES
+
 
 class BaseTreeParamName(ParamNameInterface, AbstractTreeName):
     annotation_node = None
@@ -429,7 +446,7 @@ class BaseTreeParamName(ParamNameInterface, AbstractTreeName):
 
 class _ActualTreeParamName(BaseTreeParamName):
     def __init__(self, function_value, tree_name):
-        super(_ActualTreeParamName, self).__init__(
+        super().__init__(
             function_value.get_default_param_context(), tree_name)
         self.function_value = function_value
 
@@ -499,11 +516,11 @@ class _ActualTreeParamName(BaseTreeParamName):
 class AnonymousParamName(_ActualTreeParamName):
     @plugin_manager.decorate(name='goto_anonymous_param')
     def goto(self):
-        return super(AnonymousParamName, self).goto()
+        return super().goto()
 
     @plugin_manager.decorate(name='infer_anonymous_param')
     def infer(self):
-        values = super(AnonymousParamName, self).infer()
+        values = super().infer()
         if values:
             return values
         from jedi.inference.dynamic_params import dynamic_param_lookup
@@ -527,11 +544,11 @@ class AnonymousParamName(_ActualTreeParamName):
 
 class ParamName(_ActualTreeParamName):
     def __init__(self, function_value, tree_name, arguments):
-        super(ParamName, self).__init__(function_value, tree_name)
+        super().__init__(function_value, tree_name)
         self.arguments = arguments
 
     def infer(self):
-        values = super(ParamName, self).infer()
+        values = super().infer()
         if values:
             return values
 
@@ -604,7 +621,7 @@ class SubModuleName(ImportName):
     _level = 1
 
 
-class NameWrapper(object):
+class NameWrapper:
     def __init__(self, wrapped_name):
         self._wrapped_name = wrapped_name
 
@@ -615,7 +632,7 @@ class NameWrapper(object):
         return '%s(%s)' % (self.__class__.__name__, self._wrapped_name)
 
 
-class StubNameMixin(object):
+class StubNameMixin:
     def py__doc__(self):
         from jedi.inference.gradual.conversion import convert_names
         # Stubs are not complicated and we can just follow simple statements
@@ -627,7 +644,7 @@ class StubNameMixin(object):
 
         names = convert_names(names, prefer_stub_to_compiled=False)
         if self in names:
-            return super(StubNameMixin, self).py__doc__()
+            return super().py__doc__()
         else:
             # We have signatures ourselves in stubs, so don't use signatures
             # from the implementation.
@@ -637,7 +654,7 @@ class StubNameMixin(object):
 # From here on down we make looking up the sys.version_info fast.
 class StubName(StubNameMixin, TreeNameDefinition):
     def infer(self):
-        inferred = super(StubName, self).infer()
+        inferred = super().infer()
         if self.string_name == 'version_info' and self.get_root_context().py__name__() == 'sys':
             from jedi.inference.gradual.stub_value import VersionInfo
             return ValueSet(VersionInfo(c) for c in inferred)

jedi/inference/param.py

@@ -1,4 +1,5 @@
 from collections import defaultdict
+from inspect import Parameter
 
 from jedi import debug
 from jedi.inference.utils import PushBackIterator
@@ -6,7 +7,6 @@ from jedi.inference import analysis
 from jedi.inference.lazy_value import LazyKnownValue, \
     LazyTreeValue, LazyUnknownValue
 from jedi.inference.value import iterable
-from jedi._compatibility import Parameter
 from jedi.inference.names import ParamName
 
 
@@ -20,8 +20,7 @@ def _add_argument_issue(error_name, lazy_value, message):
 
 class ExecutedParamName(ParamName):
     def __init__(self, function_value, arguments, param_node, lazy_value, is_default=False):
-        super(ExecutedParamName, self).__init__(
-            function_value, param_node.name, arguments=arguments)
+        super().__init__(function_value, param_node.name, arguments=arguments)
         self._lazy_value = lazy_value
         self._is_default = is_default
 
@@ -51,6 +50,25 @@ class ExecutedParamName(ParamName):
 
 
 def get_executed_param_names_and_issues(function_value, arguments):
+    """
+    Return a tuple of:
+      - a list of `ExecutedParamName`s corresponding to the arguments of the
+        function execution `function_value`, containing the inferred value of
+        those arguments (whether explicit or default)
+      - a list of the issues encountered while building that list
+
+    For example, given:
+    ```
+    def foo(a, b, c=None, d='d'): ...
+
+    foo(42, c='c')
+    ```
+
+    Then for the execution of `foo`, this will return a tuple containing:
+      - a list with entries for each parameter a, b, c & d; the entries for a,
+        c, & d will have their values (42, 'c' and 'd' respectively) included.
+      - a list with a single entry about the lack of a value for `b`
+    """
     def too_many_args(argument):
         m = _error_argument_count(funcdef, len(unpacked_va))
         # Just report an error for the first param that is not needed (like
@@ -207,6 +225,23 @@ def get_executed_param_names_and_issues(function_value, arguments):
 
 
 def get_executed_param_names(function_value, arguments):
+    """
+    Return a list of `ExecutedParamName`s corresponding to the arguments of the
+    function execution `function_value`, containing the inferred value of those
+    arguments (whether explicit or default). Any issues building this list (for
+    example required arguments which are missing in the invocation) are ignored.
+
+    For example, given:
+    ```
+    def foo(a, b, c=None, d='d'): ...
+
+    foo(42, c='c')
+    ```
+
+    Then for the execution of `foo`, this will return a list containing entries
+    for each parameter a, b, c & d; the entries for a, c, & d will have their
+    values (42, 'c' and 'd' respectively) included.
+    """
     return get_executed_param_names_and_issues(function_value, arguments)[0]
 
 

jedi/inference/recursion.py

@@ -3,8 +3,8 @@ Recursions are the recipe of |jedi| to conquer Python code. However, someone
 must stop recursions going mad. Some settings are here to make |jedi| stop at
 the right time. You can read more about them :ref:`here <settings-recursion>`.
 
-Next to :mod:`jedi.inference.cache` this module also makes |jedi| not
-thread-safe. Why?  ``execution_recursion_decorator`` uses class variables to
+Next to the internal ``jedi.inference.cache`` this module also makes |jedi| not
+thread-safe, because ``execution_recursion_decorator`` uses class variables to
 count the function calls.
 
 .. _settings-recursion:
@@ -12,7 +12,7 @@ count the function calls.
 Settings
 ~~~~~~~~~~
 
-Recursion settings are important if you don't want extremly
+Recursion settings are important if you don't want extremely
 recursive python code to go absolutely crazy.
 
 The default values are based on experiments while completing the |jedi| library
@@ -34,7 +34,7 @@ from jedi.inference.base_value import NO_VALUES
 
 recursion_limit = 15
 """
-Like ``sys.getrecursionlimit()``, just for |jedi|.
+Like :func:`sys.getrecursionlimit()`, just for |jedi|.
 """
 total_function_execution_limit = 200
 """
@@ -50,7 +50,7 @@ A function may not be executed more than this number of times recursively.
 """
 
 
-class RecursionDetector(object):
+class RecursionDetector:
     def __init__(self):
         self.pushed_nodes = []
 
@@ -92,7 +92,7 @@ def execution_recursion_decorator(default=NO_VALUES):
     return decorator
 
 
-class ExecutionRecursionDetector(object):
+class ExecutionRecursionDetector:
     """
     Catches recursions of executions.
     """

jedi/inference/references.py

@@ -3,12 +3,14 @@ import re
 
 from parso import python_bytes_to_unicode
 
-from jedi.file_io import KnownContentFileIO
-from jedi.inference.imports import SubModuleName, load_module_from_path
+from jedi.debug import dbg
+from jedi.file_io import KnownContentFileIO, FolderIO
+from jedi.inference.names import SubModuleName
+from jedi.inference.imports import load_module_from_path
 from jedi.inference.filters import ParserTreeFilter
 from jedi.inference.gradual.conversion import convert_names
 
-_IGNORE_FOLDERS = ('.tox', 'venv', '__pycache__')
+_IGNORE_FOLDERS = ('.tox', '.venv', '.mypy_cache', 'venv', '__pycache__')
 
 _OPENED_FILE_LIMIT = 2000
 """
@@ -37,8 +39,7 @@ def _resolve_names(definition_names, avoid_names=()):
             yield name
 
         if name.api_type == 'module':
-            for name in _resolve_names(name.goto(), definition_names):
-                yield name
+            yield from _resolve_names(name.goto(), definition_names)
 
 
 def _dictionarize(names):
@@ -89,8 +90,7 @@ def _add_names_in_same_context(context, string_name):
         names = set(filter_.get(string_name))
         if not names:
             break
-        for name in names:
-            yield name
+        yield from names
         ordered = sorted(names, key=lambda x: x.start_pos)
         until_position = ordered[0].start_pos
 
@@ -108,11 +108,10 @@ def _find_global_variables(names, search_name):
             for global_name in method().get(search_name):
                 yield global_name
                 c = module_context.create_context(global_name.tree_name)
-                for name in _add_names_in_same_context(c, global_name.string_name):
-                    yield name
+                yield from _add_names_in_same_context(c, global_name.string_name)
 
 
-def find_references(module_context, tree_name):
+def find_references(module_context, tree_name, only_in_module=False):
     inf = module_context.inference_state
     search_name = tree_name.value
 
@@ -126,10 +125,14 @@ def find_references(module_context, tree_name):
 
     found_names_dct = _dictionarize(found_names)
 
-    module_contexts = set(d.get_root_context() for d in found_names)
-    module_contexts = [module_context] + [m for m in module_contexts if m != module_context]
+    module_contexts = [module_context]
+    if not only_in_module:
+        for m in set(d.get_root_context() for d in found_names):
+            if m != module_context and m.tree_node is not None \
+                    and inf.project.path in m.py__file__().parents:
+                module_contexts.append(m)
     # For param no search for other modules is necessary.
-    if any(n.api_type == 'param' for n in found_names):
+    if only_in_module or any(n.api_type == 'param' for n in found_names):
         potential_modules = module_contexts
     else:
         potential_modules = get_module_contexts_containing_name(
@@ -156,7 +159,10 @@ def find_references(module_context, tree_name):
             else:
                 for name in new:
                     non_matching_reference_maps.setdefault(name, []).append(new)
-    return found_names_dct.values()
+    result = found_names_dct.values()
+    if only_in_module:
+        return [n for n in result if n.get_root_context() == module_context]
+    return result
 
 
 def _check_fs(inference_state, file_io, regex):
@@ -174,44 +180,68 @@ def _check_fs(inference_state, file_io, regex):
     return m.as_context()
 
 
-def gitignored_lines(folder_io, file_io):
-    ignored_paths = set()
-    ignored_names = set()
+def gitignored_paths(folder_io, file_io):
+    ignored_paths_abs = set()
+    ignored_paths_rel = set()
+
     for l in file_io.read().splitlines():
-        if not l or l.startswith(b'#'):
+        if not l or l.startswith(b'#') or l.startswith(b'!') or b'*' in l:
             continue
 
-        p = l.decode('utf-8', 'ignore')
-        if p.startswith('/'):
-            name = p[1:]
-            if name.endswith(os.path.sep):
-                name = name[:-1]
-            ignored_paths.add(os.path.join(folder_io.path, name))
+        p = l.decode('utf-8', 'ignore').rstrip('/')
+        if '/' in p:
+            name = p.lstrip('/')
+            ignored_paths_abs.add(os.path.join(folder_io.path, name))
         else:
-            ignored_names.add(p)
-    return ignored_paths, ignored_names
+            name = p
+            ignored_paths_rel.add((folder_io.path, name))
+
+    return ignored_paths_abs, ignored_paths_rel
 
 
-def _recurse_find_python_files(folder_io, except_paths):
+def expand_relative_ignore_paths(folder_io, relative_paths):
+    curr_path = folder_io.path
+    return {os.path.join(curr_path, p[1]) for p in relative_paths if curr_path.startswith(p[0])}
+
+
+def recurse_find_python_folders_and_files(folder_io, except_paths=()):
+    except_paths = set(except_paths)
+    except_paths_relative = set()
+
     for root_folder_io, folder_ios, file_ios in folder_io.walk():
         # Delete folders that we don't want to iterate over.
         for file_io in file_ios:
             path = file_io.path
-            if path.endswith('.py') or path.endswith('.pyi'):
+            if path.suffix in ('.py', '.pyi'):
                 if path not in except_paths:
-                    yield file_io
+                    yield None, file_io
 
-            if path.endswith('.gitignore'):
-                ignored_paths, ignored_names = \
-                    gitignored_lines(root_folder_io, file_io)
-                except_paths |= ignored_paths
+            if path.name == '.gitignore':
+                ignored_paths_abs, ignored_paths_rel = gitignored_paths(
+                    root_folder_io, file_io
+                )
+                except_paths |= ignored_paths_abs
+                except_paths_relative |= ignored_paths_rel
+
+        except_paths_relative_expanded = expand_relative_ignore_paths(
+            root_folder_io, except_paths_relative
+        )
 
         folder_ios[:] = [
             folder_io
             for folder_io in folder_ios
             if folder_io.path not in except_paths
+            and folder_io.path not in except_paths_relative_expanded
             and folder_io.get_base_name() not in _IGNORE_FOLDERS
         ]
+        for folder_io in folder_ios:
+            yield folder_io, None
+
+
+def recurse_find_python_files(folder_io, except_paths=()):
+    for folder_io, file_io in recurse_find_python_folders_and_files(folder_io, except_paths):
+        if file_io is not None:
+            yield file_io
 
 
 def _find_python_files_in_sys_path(inference_state, module_contexts):
@@ -228,13 +258,18 @@ def _find_python_files_in_sys_path(inference_state, module_contexts):
             path = folder_io.path
             if not any(path.startswith(p) for p in sys_path) or path in except_paths:
                 break
-            for file_io in _recurse_find_python_files(folder_io, except_paths):
+            for file_io in recurse_find_python_files(folder_io, except_paths):
                 if file_io.path not in yielded_paths:
                     yield file_io
             except_paths.add(path)
             folder_io = folder_io.get_parent_folder()
 
 
+def _find_project_modules(inference_state, module_contexts):
+    except_ = [m.py__file__() for m in module_contexts]
+    yield from recurse_find_python_files(FolderIO(inference_state.project.path), except_)
+
+
 def get_module_contexts_containing_name(inference_state, module_contexts, name,
                                         limit_reduction=1):
     """
@@ -254,19 +289,31 @@ def get_module_contexts_containing_name(inference_state, module_contexts, name,
     if len(name) <= 2:
         return
 
+    # Currently not used, because there's only `scope=project` and `scope=file`
+    # At the moment there is no such thing as `scope=sys.path`.
+    # file_io_iterator = _find_python_files_in_sys_path(inference_state, module_contexts)
+    file_io_iterator = _find_project_modules(inference_state, module_contexts)
+    yield from search_in_file_ios(inference_state, file_io_iterator, name,
+                                  limit_reduction=limit_reduction)
+
+
+def search_in_file_ios(inference_state, file_io_iterator, name,
+                       limit_reduction=1, complete=False):
     parse_limit = _PARSED_FILE_LIMIT / limit_reduction
     open_limit = _OPENED_FILE_LIMIT / limit_reduction
     file_io_count = 0
     parsed_file_count = 0
-    regex = re.compile(r'\b' + re.escape(name) + r'\b')
-    for file_io in _find_python_files_in_sys_path(inference_state, module_contexts):
+    regex = re.compile(r'\b' + re.escape(name) + (r'' if complete else r'\b'))
+    for file_io in file_io_iterator:
         file_io_count += 1
         m = _check_fs(inference_state, file_io, regex)
         if m is not None:
             parsed_file_count += 1
             yield m
             if parsed_file_count >= parse_limit:
+                dbg('Hit limit of parsed files: %s', parse_limit)
                 break
 
         if file_io_count >= open_limit:
+            dbg('Hit limit of opened files: %s', open_limit)
             break

jedi/inference/signature.py

@@ -1,10 +1,11 @@
-from jedi._compatibility import Parameter
+from inspect import Parameter
+
 from jedi.cache import memoize_method
 from jedi import debug
 from jedi import parser_utils
 
 
-class _SignatureMixin(object):
+class _SignatureMixin:
     def to_string(self):
         def param_strings():
             is_positional = False
@@ -67,7 +68,7 @@ class AbstractSignature(_SignatureMixin):
 
 class TreeSignature(AbstractSignature):
     def __init__(self, value, function_value=None, is_bound=False):
-        super(TreeSignature, self).__init__(value, is_bound)
+        super().__init__(value, is_bound)
         self._function_value = function_value or value
 
     def bind(self, value):
@@ -90,10 +91,12 @@ class TreeSignature(AbstractSignature):
 
     @memoize_method
     def get_param_names(self, resolve_stars=False):
-        params = super(TreeSignature, self).get_param_names(resolve_stars=False)
+        params = self._function_value.get_param_names()
         if resolve_stars:
             from jedi.inference.star_args import process_params
             params = process_params(params)
+        if self.is_bound:
+            return params[1:]
         return params
 
     def matches_signature(self, arguments):
@@ -119,7 +122,7 @@ class TreeSignature(AbstractSignature):
 
 class BuiltinSignature(AbstractSignature):
     def __init__(self, value, return_string, function_value=None, is_bound=False):
-        super(BuiltinSignature, self).__init__(value, is_bound)
+        super().__init__(value, is_bound)
         self._return_string = return_string
         self.__function_value = function_value
 

jedi/inference/star_args.py

@@ -10,8 +10,10 @@ This means for example in this case::
 
 The signature here for bar should be `bar(b, c)` instead of bar(*args).
 """
+from inspect import Parameter
+
+from parso import tree
 
-from jedi._compatibility import Parameter
 from jedi.inference.utils import to_list
 from jedi.inference.names import ParamNameWrapper
 from jedi.inference.helpers import is_big_annoying_library
@@ -22,7 +24,11 @@ def _iter_nodes_for_param(param_name):
     from jedi.inference.arguments import TreeArguments
 
     execution_context = param_name.parent_context
-    function_node = execution_context.tree_node
+    # Walk up the parso tree to get the FunctionNode we want. We use the parso
+    # tree rather than going via the execution context so that we're agnostic of
+    # the specific scope we're evaluating within (i.e: module or function,
+    # etc.).
+    function_node = tree.search_ancestor(param_name.tree_name, 'funcdef', 'lambdef')
     module_node = function_node.get_root_node()
     start = function_node.children[-1].start_pos
     end = function_node.children[-1].end_pos
@@ -32,8 +38,6 @@ def _iter_nodes_for_param(param_name):
             argument = name.parent
             if argument.type == 'argument' \
                     and argument.children[0] == '*' * param_name.star_count:
-                # No support for Python <= 3.4 here, but they are end-of-life
-                # anyway
                 trailer = search_ancestor(argument, 'trailer')
                 if trailer is not None:  # Make sure we're in a function
                     context = execution_context.create_context(trailer)
@@ -98,8 +102,7 @@ def process_params(param_names, star_count=3):  # default means both * and **
         if is_big_annoying_library(param_names[0].parent_context):
             # At first this feature can look innocent, but it does a lot of
             # type inference in some cases, so we just ditch it.
-            for p in param_names:
-                yield p
+            yield from param_names
             return
 
     used_names = set()
@@ -210,7 +213,7 @@ def process_params(param_names, star_count=3):  # default means both * and **
 
 class ParamNameFixedKind(ParamNameWrapper):
     def __init__(self, param_name, new_kind):
-        super(ParamNameFixedKind, self).__init__(param_name)
+        super().__init__(param_name)
         self._new_kind = new_kind
 
     def get_kind(self):

jedi/inference/syntax_tree.py

@@ -2,10 +2,10 @@
 Functions inferring the syntax tree.
 """
 import copy
+import itertools
 
 from parso.python import tree
 
-from jedi._compatibility import force_unicode, unicode
 from jedi import debug
 from jedi import parser_utils
 from jedi.inference.base_value import ValueSet, NO_VALUES, ContextualizedNode, \
@@ -31,6 +31,26 @@ from jedi.inference.context import CompForContext
 from jedi.inference.value.decorator import Decoratee
 from jedi.plugins import plugin_manager
 
+operator_to_magic_method = {
+    '+': '__add__',
+    '-': '__sub__',
+    '*': '__mul__',
+    '@': '__matmul__',
+    '/': '__truediv__',
+    '//': '__floordiv__',
+    '%': '__mod__',
+    '**': '__pow__',
+    '<<': '__lshift__',
+    '>>': '__rshift__',
+    '&': '__and__',
+    '|': '__or__',
+    '^': '__xor__',
+}
+
+reverse_operator_to_magic_method = {
+    k: '__r' + v[2:] for k, v in operator_to_magic_method.items()
+}
+
 
 def _limit_value_infers(func):
     """
@@ -205,12 +225,10 @@ def _infer_node(context, element):
                 | context.infer_node(element.children[-1]))
     elif typ == 'operator':
         # Must be an ellipsis, other operators are not inferred.
-        # In Python 2 ellipsis is coded as three single dot tokens, not
-        # as one token 3 dot token.
-        if element.value not in ('.', '...'):
+        if element.value != '...':
             origin = element.parent
             raise AssertionError("unhandled operator %s in %s " % (repr(element.value), origin))
-        return ValueSet([compiled.builtin_from_name(inference_state, u'Ellipsis')])
+        return ValueSet([compiled.builtin_from_name(inference_state, 'Ellipsis')])
     elif typ == 'dotted_name':
         value_set = infer_atom(context, element.children[0])
         for next_name in element.children[2::2]:
@@ -269,15 +287,12 @@ def infer_atom(context, atom):
     """
     state = context.inference_state
     if atom.type == 'name':
-        if atom.value in ('True', 'False', 'None'):
-            # Python 2...
-            return ValueSet([compiled.builtin_from_name(state, atom.value)])
-
         # This is the first global lookup.
-        stmt = tree.search_ancestor(
-            atom, 'expr_stmt', 'lambdef'
-        ) or atom
-        if stmt.type == 'lambdef':
+        stmt = tree.search_ancestor(atom, 'expr_stmt', 'lambdef', 'if_stmt') or atom
+        if stmt.type == 'if_stmt':
+            if not any(n.start_pos <= atom.start_pos < n.end_pos for n in stmt.get_test_nodes()):
+                stmt = atom
+        elif stmt.type == 'lambdef':
             stmt = atom
         position = stmt.start_pos
         if _is_annotation_name(atom):
@@ -292,9 +307,6 @@ def infer_atom(context, atom):
         # For False/True/None
         if atom.value in ('False', 'True', 'None'):
             return ValueSet([compiled.builtin_from_name(state, atom.value)])
-        elif atom.value == 'print':
-            # print e.g. could be inferred like this in Python 2.7
-            return NO_VALUES
         elif atom.value == 'yield':
             # Contrary to yield from, yield can just appear alone to return a
             # value when used with `.send()`.
@@ -309,7 +321,7 @@ def infer_atom(context, atom):
         value_set = infer_atom(context, atom.children[0])
         for string in atom.children[1:]:
             right = infer_atom(context, string)
-            value_set = _infer_comparison(context, value_set, u'+', right)
+            value_set = _infer_comparison(context, value_set, '+', right)
         return value_set
     elif atom.type == 'fstring':
         return compiled.get_string_value_set(state)
@@ -317,8 +329,8 @@ def infer_atom(context, atom):
         c = atom.children
         # Parentheses without commas are not tuples.
         if c[0] == '(' and not len(c) == 2 \
-                and not(c[1].type == 'testlist_comp'
-                        and len(c[1].children) > 1):
+                and not (c[1].type == 'testlist_comp'
+                         and len(c[1].children) > 1):
             return context.infer_node(c[1])
 
         try:
@@ -356,6 +368,12 @@ def infer_atom(context, atom):
 def infer_expr_stmt(context, stmt, seek_name=None):
     with recursion.execution_allowed(context.inference_state, stmt) as allowed:
         if allowed:
+            if seek_name is not None:
+                pep0484_values = \
+                    annotation.find_type_from_comment_hint_assign(context, stmt, seek_name)
+                if pep0484_values:
+                    return pep0484_values
+
             return _infer_expr_stmt(context, stmt, seek_name)
     return NO_VALUES
 
@@ -388,6 +406,7 @@ def _infer_expr_stmt(context, stmt, seek_name=None):
 
     debug.dbg('infer_expr_stmt %s (%s)', stmt, seek_name)
     rhs = stmt.get_rhs()
+
     value_set = context.infer_node(rhs)
 
     if seek_name:
@@ -474,8 +493,10 @@ def infer_factor(value_set, operator):
         elif operator == 'not':
             b = value.py__bool__()
             if b is None:  # Uncertainty.
-                return
-            yield compiled.create_simple_object(value.inference_state, not b)
+                yield list(value.inference_state.builtins_module.py__getattribute__('bool')
+                           .execute_annotation()).pop()
+            else:
+                yield compiled.create_simple_object(value.inference_state, not b)
         else:
             yield value
 
@@ -497,10 +518,20 @@ def _literals_to_types(inference_state, result):
 
 def _infer_comparison(context, left_values, operator, right_values):
     state = context.inference_state
+    if isinstance(operator, str):
+        operator_str = operator
+    else:
+        operator_str = str(operator.value)
     if not left_values or not right_values:
         # illegal slices e.g. cause left/right_result to be None
         result = (left_values or NO_VALUES) | (right_values or NO_VALUES)
         return _literals_to_types(state, result)
+    elif operator_str == "|" and all(
+        value.is_class() or value.is_compiled()
+        for value in itertools.chain(left_values, right_values)
+    ):
+        # ^^^ A naive hack for PEP 604
+        return ValueSet.from_sets((left_values, right_values))
     else:
         # I don't think there's a reasonable chance that a string
         # operation is still correct, once we pass something like six
@@ -531,16 +562,16 @@ def _is_annotation_name(name):
     return False
 
 
-def _is_tuple(value):
-    return isinstance(value, iterable.Sequence) and value.array_type == 'tuple'
-
-
 def _is_list(value):
-    return isinstance(value, iterable.Sequence) and value.array_type == 'list'
+    return value.array_type == 'list'
+
+
+def _is_tuple(value):
+    return value.array_type == 'tuple'
 
 
 def _bool_to_value(inference_state, bool_):
-    return compiled.builtin_from_name(inference_state, force_unicode(str(bool_)))
+    return compiled.builtin_from_name(inference_state, str(bool_))
 
 
 def _get_tuple_ints(value):
@@ -563,10 +594,10 @@ def _get_tuple_ints(value):
 def _infer_comparison_part(inference_state, context, left, operator, right):
     l_is_num = is_number(left)
     r_is_num = is_number(right)
-    if isinstance(operator, unicode):
+    if isinstance(operator, str):
         str_operator = operator
     else:
-        str_operator = force_unicode(str(operator.value))
+        str_operator = str(operator.value)
 
     if str_operator == '*':
         # for iterables, ignore * operations
@@ -577,7 +608,7 @@ def _infer_comparison_part(inference_state, context, left, operator, right):
     elif str_operator == '+':
         if l_is_num and r_is_num or is_string(left) and is_string(right):
             return left.execute_operation(right, str_operator)
-        elif _is_tuple(left) and _is_tuple(right) or _is_list(left) and _is_list(right):
+        elif _is_list(left) and _is_list(right) or _is_tuple(left) and _is_tuple(right):
             return ValueSet([iterable.MergedArray(inference_state, (left, right))])
     elif str_operator == '-':
         if l_is_num and r_is_num:
@@ -596,7 +627,11 @@ def _infer_comparison_part(inference_state, context, left, operator, right):
             if str_operator in ('is', '!=', '==', 'is not'):
                 operation = COMPARISON_OPERATORS[str_operator]
                 bool_ = operation(left, right)
-                return ValueSet([_bool_to_value(inference_state, bool_)])
+                # Only if == returns True or != returns False, we can continue.
+                # There's no guarantee that they are not equal. This can help
+                # in some cases, but does not cover everything.
+                if (str_operator in ('is', '==')) == bool_:
+                    return ValueSet([_bool_to_value(inference_state, bool_)])
 
             if isinstance(left, VersionInfo):
                 version_info = _get_tuple_ints(right)
@@ -611,8 +646,8 @@ def _infer_comparison_part(inference_state, context, left, operator, right):
             _bool_to_value(inference_state, True),
             _bool_to_value(inference_state, False)
         ])
-    elif str_operator == 'in':
-        return NO_VALUES
+    elif str_operator in ('in', 'not in'):
+        return inference_state.builtins_module.py__getattribute__('bool').execute_annotation()
 
     def check(obj):
         """Checks if a Jedi object is either a float or an int."""
@@ -626,28 +661,29 @@ def _infer_comparison_part(inference_state, context, left, operator, right):
         analysis.add(context, 'type-error-operation', operator,
                      message % (left, right))
 
+    if left.is_class() or right.is_class():
+        return NO_VALUES
+
+    method_name = operator_to_magic_method[str_operator]
+    magic_methods = left.py__getattribute__(method_name)
+    if magic_methods:
+        result = magic_methods.execute_with_values(right)
+        if result:
+            return result
+
+    if not magic_methods:
+        reverse_method_name = reverse_operator_to_magic_method[str_operator]
+        magic_methods = right.py__getattribute__(reverse_method_name)
+
+        result = magic_methods.execute_with_values(left)
+        if result:
+            return result
+
     result = ValueSet([left, right])
     debug.dbg('Used operator %s resulting in %s', operator, result)
     return result
 
 
-def _remove_statements(context, stmt, name):
-    """
-    This is the part where statements are being stripped.
-
-    Due to lazy type inference, statements like a = func; b = a; b() have to be
-    inferred.
-
-    TODO merge with infer_expr_stmt?
-    """
-    pep0484_values = \
-        annotation.find_type_from_comment_hint_assign(context, stmt, name)
-    if pep0484_values:
-        return pep0484_values
-
-    return infer_expr_stmt(context, stmt, seek_name=name)
-
-
 @plugin_manager.decorate()
 def tree_name_to_values(inference_state, context, tree_name):
     value_set = NO_VALUES
@@ -655,16 +691,25 @@ def tree_name_to_values(inference_state, context, tree_name):
     # First check for annotations, like: `foo: int = 3`
     if module_node is not None:
         names = module_node.get_used_names().get(tree_name.value, [])
+        found_annotation = False
         for name in names:
             expr_stmt = name.parent
 
             if expr_stmt.type == "expr_stmt" and expr_stmt.children[1].type == "annassign":
                 correct_scope = parser_utils.get_parent_scope(name) == context.tree_node
+                ann_assign = expr_stmt.children[1]
                 if correct_scope:
+                    found_annotation = True
+                    if (
+                        (ann_assign.children[1].type == 'name')
+                        and (ann_assign.children[1].value == tree_name.value)
+                        and context.parent_context
+                    ):
+                        context = context.parent_context
                     value_set |= annotation.infer_annotation(
                         context, expr_stmt.children[1].children[1]
                     ).execute_annotation()
-        if value_set:
+        if found_annotation:
             return value_set
 
     types = []
@@ -710,10 +755,17 @@ def tree_name_to_values(inference_state, context, tree_name):
             n = TreeNameDefinition(context, tree_name)
             types = check_tuple_assignments(n, for_types)
     elif typ == 'expr_stmt':
-        types = _remove_statements(context, node, tree_name)
+        types = infer_expr_stmt(context, node, tree_name)
     elif typ == 'with_stmt':
         value_managers = context.infer_node(node.get_test_node_from_name(tree_name))
-        enter_methods = value_managers.py__getattribute__(u'__enter__')
+        if node.parent.type == 'async_stmt':
+            # In the case of `async with` statements, we need to
+            # first get the coroutine from the `__aenter__` method,
+            # then "unwrap" via the `__await__` method
+            enter_methods = value_managers.py__getattribute__('__aenter__')
+            coro = enter_methods.execute_with_values()
+            return coro.py__await__().py__stop_iteration_returns()
+        enter_methods = value_managers.py__getattribute__('__enter__')
         return enter_methods.execute_with_values()
     elif typ in ('import_from', 'import_name'):
         types = imports.infer_import(context, tree_name)
@@ -729,6 +781,8 @@ def tree_name_to_values(inference_state, context, tree_name):
         types = NO_VALUES
     elif typ == 'del_stmt':
         types = NO_VALUES
+    elif typ == 'namedexpr_test':
+        types = infer_node(context, node)
     else:
         raise ValueError("Should not happen. type: %s" % typ)
     return types
@@ -797,7 +851,8 @@ def check_tuple_assignments(name, value_set):
         if isinstance(index, slice):
             # For no star unpacking is not possible.
             return NO_VALUES
-        for _ in range(index + 1):
+        i = 0
+        while i <= index:
             try:
                 lazy_value = next(iterated)
             except StopIteration:
@@ -806,6 +861,8 @@ def check_tuple_assignments(name, value_set):
                 # index number is high. Therefore break if the loop is
                 # finished.
                 return NO_VALUES
+            else:
+                i += lazy_value.max
         value_set = lazy_value.infer()
     return value_set
 
@@ -824,8 +881,7 @@ def _infer_subscript_list(context, index):
         return ValueSet([iterable.Slice(context, None, None, None)])
 
     elif index.type == 'subscript' and not index.children[0] == '.':
-        # subscript basically implies a slice operation, except for Python 2's
-        # Ellipsis.
+        # subscript basically implies a slice operation
         # e.g. array[:3]
         result = []
         for el in index.children: