Add a new release 0.13.2

Use `pytest.param` when marking single parameters

.travis.yml

@@ -8,7 +8,6 @@ python:
 
 env:
   - JEDI_TEST_ENVIRONMENT=27
-  - JEDI_TEST_ENVIRONMENT=33
   - JEDI_TEST_ENVIRONMENT=34
   - JEDI_TEST_ENVIRONMENT=35
   - JEDI_TEST_ENVIRONMENT=36
@@ -39,7 +38,6 @@ before_install:
     - ./travis_install.sh
     # Need to add the path to the Python versions in the end. This might add
     # something twice, but it doesn't really matter, because they are appended.
-    - export PATH=$PATH:/opt/python/3.3/bin
     - export PATH=$PATH:/opt/python/3.5/bin
     # 3.6 was not installed manually, but already is on the system. However
     # it's not on path (unless 3.6 is selected).

CHANGELOG.rst

@@ -3,6 +3,28 @@
 Changelog
 ---------
 
+0.13.2 (2018-12-15)
++++++++++++++++++++
+
+- Fixed a bug that led to Jedi spawning a lot of subprocesses.
+
+0.13.1 (2018-10-02)
++++++++++++++++++++
+
+- Bugfixes, because tensorflow completions were still slow.
+
+0.13.0 (2018-10-02)
++++++++++++++++++++
+
+- A small release. Some bug fixes.
+- Remove Python 3.3 support. Python 3.3 support has been dropped by the Python
+  foundation.
+- Default environments are now using the same Python version as the Python
+  process. In 0.12.x, we used to load the latest Python version on the system.
+- Added ``include_builtins`` as a parameter to usages.
+- ``goto_assignments`` has a new ``follow_builtin_imports`` parameter that
+  changes the previous behavior slightly.
+
 0.12.1 (2018-06-30)
 +++++++++++++++++++
 

CONTRIBUTING.md

@@ -5,4 +5,4 @@ Pull Requests are great.
  3. Add your name to AUTHORS.txt
  4. Push to your fork and submit a pull request.
 
-**Try to use the PEP8 style guide.**
+**Try to use the PEP8 style guide** (and it's ok to have a line length of 100 characters).

README.rst

@@ -111,8 +111,8 @@ understands, see: `Features
 <https://jedi.readthedocs.org/en/latest/docs/features.html>`_. A list of
 caveats can be found on the same page.
 
-You can run Jedi on CPython 2.7 or 3.3+ but it should also
-understand/parse code older than those versions. Additonally you should be able
+You can run Jedi on CPython 2.7 or 3.4+ but it should also
+understand/parse code older than those versions. Additionally you should be able
 to use `Virtualenvs <https://jedi.readthedocs.org/en/latest/docs/api.html#environments>`_
 very well.
 
@@ -122,7 +122,7 @@ Tips on how to use Jedi efficiently can be found `here
 API
 ---
 
-You can find the documentation for the `API here <https://jedi.readthedocs.org/en/latest/docs/plugin-api.html>`_.
+You can find the documentation for the `API here <https://jedi.readthedocs.org/en/latest/docs/api.html>`_.
 
 
 Autocompletion / Goto / Pydoc

appveyor.yml

@@ -3,9 +3,6 @@ environment:
   - TOXENV: py27
     PYTHON_PATH: C:\Python27
     JEDI_TEST_ENVIRONMENT: 27
-  - TOXENV: py27
-    PYTHON_PATH: C:\Python27
-    JEDI_TEST_ENVIRONMENT: 33
   - TOXENV: py27
     PYTHON_PATH: C:\Python27
     JEDI_TEST_ENVIRONMENT: 34
@@ -16,28 +13,9 @@ environment:
     PYTHON_PATH: C:\Python27
     JEDI_TEST_ENVIRONMENT: 36
 
-  - TOXENV: py33
-    PYTHON_PATH: C:\Python33
-    JEDI_TEST_ENVIRONMENT: 27
-  - TOXENV: py33
-    PYTHON_PATH: C:\Python33
-    JEDI_TEST_ENVIRONMENT: 33
-  - TOXENV: py33
-    PYTHON_PATH: C:\Python33
-    JEDI_TEST_ENVIRONMENT: 34
-  - TOXENV: py33
-    PYTHON_PATH: C:\Python33
-    JEDI_TEST_ENVIRONMENT: 35
-  - TOXENV: py33
-    PYTHON_PATH: C:\Python33
-    JEDI_TEST_ENVIRONMENT: 36
-
   - TOXENV: py34
     PYTHON_PATH: C:\Python34
     JEDI_TEST_ENVIRONMENT: 27
-  - TOXENV: py34
-    PYTHON_PATH: C:\Python34
-    JEDI_TEST_ENVIRONMENT: 33
   - TOXENV: py34
     PYTHON_PATH: C:\Python34
     JEDI_TEST_ENVIRONMENT: 34
@@ -51,9 +29,6 @@ environment:
   - TOXENV: py35
     PYTHON_PATH: C:\Python35
     JEDI_TEST_ENVIRONMENT: 27
-  - TOXENV: py35
-    PYTHON_PATH: C:\Python35
-    JEDI_TEST_ENVIRONMENT: 33
   - TOXENV: py35
     PYTHON_PATH: C:\Python35
     JEDI_TEST_ENVIRONMENT: 34
@@ -67,9 +42,6 @@ environment:
   - TOXENV: py36
     PYTHON_PATH: C:\Python36
     JEDI_TEST_ENVIRONMENT: 27
-  - TOXENV: py36
-    PYTHON_PATH: C:\Python36
-    JEDI_TEST_ENVIRONMENT: 33
   - TOXENV: py36
     PYTHON_PATH: C:\Python36
     JEDI_TEST_ENVIRONMENT: 34

conftest.py

@@ -6,7 +6,7 @@ from functools import partial
 import pytest
 
 import jedi
-from jedi.api.environment import get_default_environment, get_system_environment
+from jedi.api.environment import get_system_environment, InterpreterEnvironment
 from jedi._compatibility import py_version
 
 collect_ignore = [
@@ -41,6 +41,9 @@ def pytest_addoption(parser):
 
     parser.addoption("--env", action='store',
                      help="Execute the tests in that environment (e.g. 35 for python3.5).")
+    parser.addoption("--interpreter-env", "-I", action='store_true',
+                     help="Don't use subprocesses to guarantee having safe "
+                          "code execution. Useful for debugging.")
 
 
 def pytest_configure(config):
@@ -87,13 +90,13 @@ def clean_jedi_cache(request):
 
 @pytest.fixture(scope='session')
 def environment(request):
+    if request.config.option.interpreter_env:
+        return InterpreterEnvironment()
+
     version = request.config.option.env
     if version is None:
         version = os.environ.get('JEDI_TEST_ENVIRONMENT', str(py_version))
 
-    if int(version) == py_version:
-        return get_default_environment()
-
     return get_system_environment(version[0] + '.' + version[1:])
 
 

docs/docs/features.rst

@@ -20,7 +20,7 @@ make it work.
 General Features
 ----------------
 
-- Python 2.7 and 3.3+ support
+- Python 2.7 and 3.4+ support
 - Ignores syntax errors and wrong indentation
 - Can deal with complex module / function / class structures
 - Great Virtualenv support

jedi/__init__.py

@@ -23,7 +23,7 @@ example for the autocompletion feature:
 ... datetime.da'''
 >>> script = jedi.Script(source, 3, len('datetime.da'), 'example.py')
 >>> script
-<Script: 'example.py'>
+<Script: 'example.py' ...>
 >>> completions = script.completions()
 >>> completions                                         #doctest: +ELLIPSIS
 [<Completion: date>, <Completion: datetime>, ...]
@@ -36,7 +36,7 @@ As you see Jedi is pretty simple and allows you to concentrate on writing a
 good text editor, while still having very good IDE features for Python.
 """
 
-__version__ = '0.12.1'
+__version__ = '0.13.2'
 
 from jedi.api import Script, Interpreter, set_debug_function, \
     preload_module, names

jedi/_compatibility.py

@@ -16,8 +16,6 @@ except ImportError:
     pass
 
 is_py3 = sys.version_info[0] >= 3
-is_py33 = is_py3 and sys.version_info[1] >= 3
-is_py34 = is_py3 and sys.version_info[1] >= 4
 is_py35 = is_py3 and sys.version_info[1] >= 5
 py_version = int(str(sys.version_info[0]) + str(sys.version_info[1]))
 
@@ -34,24 +32,36 @@ class DummyFile(object):
         del self.loader
 
 
-def find_module_py34(string, path=None, full_name=None):
+def find_module_py34(string, path=None, full_name=None, is_global_search=True):
     spec = None
     loader = None
 
-    spec = importlib.machinery.PathFinder.find_spec(string, path)
-    if spec is not None:
-        # We try to disambiguate implicit namespace pkgs with non implicit namespace pkgs
-        if not spec.has_location:
-            full_name = string if not path else full_name
-            implicit_ns_info = ImplicitNSInfo(full_name, spec.submodule_search_locations._path)
-            return None, implicit_ns_info, False
+    for finder in sys.meta_path:
+        if is_global_search and finder != importlib.machinery.PathFinder:
+            p = None
+        else:
+            p = path
+        try:
+            find_spec = finder.find_spec
+        except AttributeError:
+            # These are old-school clases that still have a different API, just
+            # ignore those.
+            continue
+
+        spec = find_spec(string, p)
+        if spec is not None:
+            loader = spec.loader
+            if loader is None and not spec.has_location:
+                # This is a namespace package.
+                full_name = string if not path else full_name
+                implicit_ns_info = ImplicitNSInfo(full_name, spec.submodule_search_locations._path)
+                return None, implicit_ns_info, False
+            break
 
-        # we have found the tail end of the dotted path
-        loader = spec.loader
     return find_module_py33(string, path, loader)
 
 
-def find_module_py33(string, path=None, loader=None, full_name=None):
+def find_module_py33(string, path=None, loader=None, full_name=None, is_global_search=True):
     loader = loader or importlib.machinery.PathFinder.find_module(string, path)
 
     if loader is None and path is None:  # Fallback to find builtins
@@ -104,7 +114,7 @@ def find_module_py33(string, path=None, loader=None, full_name=None):
     return module_file, module_path, is_package
 
 
-def find_module_pre_py33(string, path=None, full_name=None):
+def find_module_pre_py34(string, path=None, full_name=None, is_global_search=True):
     # This import is here, because in other places it will raise a
     # DeprecationWarning.
     import imp
@@ -139,8 +149,7 @@ def find_module_pre_py33(string, path=None, full_name=None):
     raise ImportError("No module named {}".format(string))
 
 
-find_module = find_module_py33 if is_py33 else find_module_pre_py33
-find_module = find_module_py34 if is_py34 else find_module
+find_module = find_module_py34 if is_py3 else find_module_pre_py34
 find_module.__doc__ = """
 Provides information about a module.
 
@@ -207,6 +216,7 @@ def _iter_modules(paths, prefix=''):
                 yield importer, prefix + modname, ispkg
         # END COPY
 
+
 iter_modules = _iter_modules if py_version >= 34 else pkgutil.iter_modules
 
 
@@ -252,6 +262,7 @@ Usage::
 
 """
 
+
 class Python3Method(object):
     def __init__(self, func):
         self.func = func
@@ -312,10 +323,10 @@ def force_unicode(obj):
 try:
     import builtins  # module name in python 3
 except ImportError:
-    import __builtin__ as builtins
+    import __builtin__ as builtins  # noqa: F401
 
 
-import ast
+import ast  # noqa: F401
 
 
 def literal_eval(string):
@@ -325,7 +336,7 @@ def literal_eval(string):
 try:
     from itertools import zip_longest
 except ImportError:
-    from itertools import izip_longest as zip_longest  # Python 2
+    from itertools import izip_longest as zip_longest  # Python 2  # noqa: F401
 
 try:
     FileNotFoundError = FileNotFoundError
@@ -355,6 +366,7 @@ def print_to_stderr(*args):
         eval("print(*args, file=sys.stderr)")
     else:
         print >> sys.stderr, args
+    sys.stderr.flush()
 
 
 def utf8_repr(func):
@@ -378,7 +390,7 @@ def utf8_repr(func):
 if is_py3:
     import queue
 else:
-    import Queue as queue
+    import Queue as queue  # noqa: F401
 
 try:
     # Attempt to load the C implementation of pickle on Python 2 as it is way
@@ -507,6 +519,9 @@ class GeneralizedPopen(subprocess.Popen):
             except AttributeError:
                 CREATE_NO_WINDOW = 0x08000000
             kwargs['creationflags'] = CREATE_NO_WINDOW
+        # The child process doesn't need file descriptors except 0, 1, 2.
+        # This is unix only.
+        kwargs['close_fds'] = 'posix' in sys.builtin_module_names
         super(GeneralizedPopen, self).__init__(*args, **kwargs)
 
 
@@ -544,7 +559,7 @@ def which(cmd, mode=os.F_OK | os.X_OK, path=None):
 
     if sys.platform == "win32":
         # The current directory takes precedence on Windows.
-        if not os.curdir in path:
+        if os.curdir not in path:
             path.insert(0, os.curdir)
 
         # PATHEXT is necessary to check on Windows.
@@ -565,7 +580,7 @@ def which(cmd, mode=os.F_OK | os.X_OK, path=None):
     seen = set()
     for dir in path:
         normdir = os.path.normcase(dir)
-        if not normdir in seen:
+        if normdir not in seen:
             seen.add(normdir)
             for thefile in files:
                 name = os.path.join(dir, thefile)

jedi/api/__init__.py

@@ -11,6 +11,7 @@ arguments.
 """
 import os
 import sys
+import warnings
 
 import parso
 from parso.python import tree
@@ -74,9 +75,6 @@ class Script(object):
     :param encoding: The encoding of ``source``, if it is not a
         ``unicode`` object (default ``'utf-8'``).
     :type encoding: str
-    :param source_encoding: The encoding of ``source``, if it is not a
-        ``unicode`` object (default ``'utf-8'``).
-    :type encoding: str
     :param sys_path: ``sys.path`` to use during analysis of the script
     :type sys_path: list
     :param environment: TODO
@@ -114,9 +112,10 @@ class Script(object):
         self._module_node, source = self._evaluator.parse_and_get_code(
             code=source,
             path=self.path,
+            encoding=encoding,
             cache=False,  # No disk cache, because the current script often changes.
-            diff_cache=True,
-            cache_path=settings.cache_directory
+            diff_cache=settings.fast_parser,
+            cache_path=settings.cache_directory,
         )
         debug.speed('parsed')
         self._code_lines = parso.split_lines(source, keepends=True)
@@ -134,7 +133,9 @@ class Script(object):
 
         column = line_len if column is None else column
         if not (0 <= column <= line_len):
-            raise ValueError('`column` parameter is not in a valid range.')
+            raise ValueError('`column` parameter (%d) is not in a valid range '
+                             '(0-%d) for line %d (%r).' % (
+                                 column, line_len, line, line_string))
         self._pos = line, column
         self._path = path
 
@@ -144,9 +145,9 @@ class Script(object):
     def _get_module(self):
         name = '__main__'
         if self.path is not None:
-            n = dotted_path_in_sys_path(self._evaluator.get_sys_path(), self.path)
-            if n is not None:
-                name = n
+            import_names = dotted_path_in_sys_path(self._evaluator.get_sys_path(), self.path)
+            if import_names is not None:
+                name = '.'.join(import_names)
 
         module = ModuleContext(
             self._evaluator, self._module_node, self.path,
@@ -156,7 +157,11 @@ class Script(object):
         return module
 
     def __repr__(self):
-        return '<%s: %s>' % (self.__class__.__name__, repr(self._orig_path))
+        return '<%s: %s %r>' % (
+            self.__class__.__name__,
+            repr(self._orig_path),
+            self._evaluator.environment,
+        )
 
     def completions(self):
         """
@@ -172,6 +177,24 @@ class Script(object):
             self._pos, self.call_signatures
         )
         completions = completion.completions()
+
+        def iter_import_completions():
+            for c in completions:
+                tree_name = c._name.tree_name
+                if tree_name is None:
+                    continue
+                definition = tree_name.get_definition()
+                if definition is not None \
+                        and definition.type in ('import_name', 'import_from'):
+                    yield c
+
+        if len(list(iter_import_completions())) > 10:
+            # For now disable completions if there's a lot of imports that
+            # might potentially be resolved. This is the case for tensorflow
+            # and has been fixed for it. This is obviously temporary until we
+            # have a better solution.
+            self._evaluator.infer_enabled = False
+
         debug.speed('completions end')
         return completions
 
@@ -203,20 +226,33 @@ class Script(object):
         # the API.
         return helpers.sorted_definitions(set(defs))
 
-    def goto_assignments(self, follow_imports=False):
+    def goto_assignments(self, follow_imports=False, follow_builtin_imports=False):
         """
         Return the first definition found, while optionally following imports.
         Multiple objects may be returned, because Python itself is a
         dynamic language, which means depending on an option you can have two
         different versions of a function.
 
+        :param follow_imports: The goto call will follow imports.
+        :param follow_builtin_imports: If follow_imports is True will decide if
+            it follow builtin imports.
         :rtype: list of :class:`classes.Definition`
         """
         def filter_follow_imports(names, check):
             for name in names:
                 if check(name):
-                    for result in filter_follow_imports(name.goto(), check):
-                        yield result
+                    new_names = list(filter_follow_imports(name.goto(), check))
+                    found_builtin = False
+                    if follow_builtin_imports:
+                        for new_name in new_names:
+                            if new_name.start_pos is None:
+                                found_builtin = True
+
+                    if found_builtin and not isinstance(name, imports.SubModuleName):
+                        yield name
+                    else:
+                        for new_name in new_names:
+                            yield new_name
                 else:
                     yield name
 
@@ -238,7 +274,7 @@ class Script(object):
         defs = [classes.Definition(self._evaluator, d) for d in set(names)]
         return helpers.sorted_definitions(defs)
 
-    def usages(self, additional_module_paths=()):
+    def usages(self, additional_module_paths=(), **kwargs):
         """
         Return :class:`classes.Definition` objects, which contain all
         names that point to the definition of the name under the cursor. This
@@ -247,17 +283,31 @@ class Script(object):
 
         .. todo:: Implement additional_module_paths
 
+        :param additional_module_paths: Deprecated, never ever worked.
+        :param include_builtins: Default True, checks if a usage is a builtin
+            (e.g. ``sys``) and in that case does not return it.
         :rtype: list of :class:`classes.Definition`
         """
-        tree_name = self._module_node.get_name_of_position(self._pos)
-        if tree_name is None:
-            # Must be syntax
-            return []
+        if additional_module_paths:
+            warnings.warn(
+                "Deprecated since version 0.12.0. This never even worked, just ignore it.",
+                DeprecationWarning,
+                stacklevel=2
+            )
 
-        names = usages.usages(self._get_module(), tree_name)
+        def _usages(include_builtins=True):
+            tree_name = self._module_node.get_name_of_position(self._pos)
+            if tree_name is None:
+                # Must be syntax
+                return []
 
-        definitions = [classes.Definition(self._evaluator, n) for n in names]
-        return helpers.sorted_definitions(definitions)
+            names = usages.usages(self._get_module(), tree_name)
+
+            definitions = [classes.Definition(self._evaluator, n) for n in names]
+            if not include_builtins:
+                definitions = [d for d in definitions if not d.in_builtin_module()]
+            return helpers.sorted_definitions(definitions)
+        return _usages(**kwargs)
 
     def call_signatures(self):
         """

jedi/api/classes.py

@@ -14,7 +14,7 @@ from jedi.evaluate import imports
 from jedi.evaluate import compiled
 from jedi.evaluate.imports import ImportName
 from jedi.evaluate.context import instance
-from jedi.evaluate.context import ClassContext, FunctionContext, FunctionExecutionContext
+from jedi.evaluate.context import ClassContext, FunctionExecutionContext
 from jedi.api.keywords import KeywordName
 
 
@@ -353,10 +353,7 @@ class BaseDefinition(object):
             return None
 
         if isinstance(context, FunctionExecutionContext):
-            # TODO the function context should be a part of the function
-            # execution context.
-            context = FunctionContext(
-                self._evaluator, context.parent_context, context.tree_node)
+            context = context.function_context
         return Definition(self._evaluator, context.name)
 
     def __repr__(self):
@@ -536,9 +533,9 @@ class Definition(BaseDefinition):
         # here.
         txt = definition.get_code(include_prefix=False)
         # Delete comments:
-        txt = re.sub('#[^\n]+\n', ' ', txt)
+        txt = re.sub(r'#[^\n]+\n', ' ', txt)
         # Delete multi spaces/newlines
-        txt = re.sub('\s+', ' ', txt).strip()
+        txt = re.sub(r'\s+', ' ', txt).strip()
         return txt
 
     @property
@@ -638,9 +635,18 @@ class CallSignature(Definition):
         """
         return self._bracket_start_pos
 
+    @property
+    def _params_str(self):
+        return ', '.join([p.description[6:]
+                          for p in self.params])
+
     def __repr__(self):
-        return '<%s: %s index %s>' % \
-            (type(self).__name__, self._name.string_name, self.index)
+        return '<%s: %s index=%r params=[%s]>' % (
+            type(self).__name__,
+            self._name.string_name,
+            self._index,
+            self._params_str,
+        )
 
 
 class _Help(object):

jedi/api/environment.py

@@ -3,23 +3,21 @@ Environments are a way to activate different Python versions or Virtualenvs for
 static analysis. The Python binary in that environment is going to be executed.
 """
 import os
-import re
 import sys
 import hashlib
 import filecmp
-from subprocess import PIPE
 from collections import namedtuple
 
-from jedi._compatibility import GeneralizedPopen, which
+from jedi._compatibility import highest_pickle_protocol, which
 from jedi.cache import memoize_method, time_cache
-from jedi.evaluate.compiled.subprocess import get_subprocess, \
+from jedi.evaluate.compiled.subprocess import CompiledSubprocess, \
     EvaluatorSameProcess, EvaluatorSubprocess
 
 import parso
 
 _VersionInfo = namedtuple('VersionInfo', 'major minor micro')
 
-_SUPPORTED_PYTHONS = ['3.6', '3.5', '3.4', '3.3', '2.7']
+_SUPPORTED_PYTHONS = ['3.7', '3.6', '3.5', '3.4', '3.3', '2.7']
 _SAFE_PATHS = ['/usr/bin', '/usr/local/bin']
 _CURRENT_VERSION = '%s.%s' % (sys.version_info.major, sys.version_info.minor)
 
@@ -46,49 +44,66 @@ class _BaseEnvironment(object):
             return self._hash
 
 
+def _get_info():
+    return (
+        sys.executable,
+        sys.prefix,
+        sys.version_info[:3],
+    )
+
+
 class Environment(_BaseEnvironment):
     """
     This class is supposed to be created by internal Jedi architecture. You
     should not create it directly. Please use create_environment or the other
     functions instead. It is then returned by that function.
     """
-    def __init__(self, path, executable):
-        self.path = os.path.abspath(path)
-        """
-        The path to an environment, matches ``sys.prefix``.
-        """
-        self.executable = os.path.abspath(executable)
+    _subprocess = None
+
+    def __init__(self, executable):
+        self._start_executable = executable
+        # Initialize the environment
+        self._get_subprocess()
+
+    def _get_subprocess(self):
+        if self._subprocess is not None and not self._subprocess.is_crashed:
+            return self._subprocess
+
+        try:
+            self._subprocess = CompiledSubprocess(self._start_executable)
+            info = self._subprocess._send(None, _get_info)
+        except Exception as exc:
+            raise InvalidPythonEnvironment(
+                "Could not get version information for %r: %r" % (
+                    self._start_executable,
+                    exc))
+
+        # Since it could change and might not be the same(?) as the one given,
+        # set it here.
+        self.executable = info[0]
         """
         The Python executable, matches ``sys.executable``.
         """
-        self.version_info = self._get_version()
+        self.path = info[1]
+        """
+        The path to an environment, matches ``sys.prefix``.
+        """
+        self.version_info = _VersionInfo(*info[2])
         """
-
         Like ``sys.version_info``. A tuple to show the current Environment's
         Python version.
         """
 
-    def _get_version(self):
-        try:
-            process = GeneralizedPopen([self.executable, '--version'], stdout=PIPE, stderr=PIPE)
-            stdout, stderr = process.communicate()
-            retcode = process.poll()
-            if retcode:
-                raise InvalidPythonEnvironment(
-                    "Exited with %d (stdout=%r, stderr=%r)" % (
-                        retcode, stdout, stderr))
-        except OSError as exc:
-            raise InvalidPythonEnvironment(
-                "Could not get version information: %r" % exc)
+        # py2 sends bytes via pickle apparently?!
+        if self.version_info.major == 2:
+            self.executable = self.executable.decode()
+            self.path = self.path.decode()
 
-        # Until Python 3.4 wthe version string is part of stderr, after that
-        # stdout.
-        output = stdout + stderr
-        match = re.match(br'Python (\d+)\.(\d+)\.(\d+)', output)
-        if match is None:
-            raise InvalidPythonEnvironment("--version not working")
+        # Adjust pickle protocol according to host and client version.
+        self._subprocess._pickle_protocol = highest_pickle_protocol([
+            sys.version_info, self.version_info])
 
-        return _VersionInfo(*[int(m) for m in match.groups()])
+        return self._subprocess
 
     def __repr__(self):
         version = '.'.join(str(i) for i in self.version_info)
@@ -97,9 +112,6 @@ class Environment(_BaseEnvironment):
     def get_evaluator_subprocess(self, evaluator):
         return EvaluatorSubprocess(evaluator, self._get_subprocess())
 
-    def _get_subprocess(self):
-        return get_subprocess(self.executable, self.version_info)
-
     @memoize_method
     def get_sys_path(self):
         """
@@ -118,10 +130,9 @@ class Environment(_BaseEnvironment):
 
 class SameEnvironment(Environment):
     def __init__(self):
-        super(SameEnvironment, self).__init__(sys.prefix, sys.executable)
-
-    def _get_version(self):
-        return _VersionInfo(*sys.version_info[:3])
+        self._start_executable = self.executable = sys.executable
+        self.path = sys.prefix
+        self.version_info = _VersionInfo(*sys.version_info[:3])
 
 
 class InterpreterEnvironment(_BaseEnvironment):
@@ -136,13 +147,18 @@ class InterpreterEnvironment(_BaseEnvironment):
 
 
 def _get_virtual_env_from_var():
+    """Get virtualenv environment from VIRTUAL_ENV environment variable.
+
+    It uses `safe=False` with ``create_environment``, because the environment
+    variable is considered to be safe / controlled by the user solely.
+    """
     var = os.environ.get('VIRTUAL_ENV')
     if var is not None:
         if var == sys.prefix:
             return SameEnvironment()
 
         try:
-            return create_environment(var)
+            return create_environment(var, safe=False)
         except InvalidPythonEnvironment:
             pass
 
@@ -168,16 +184,22 @@ def get_default_environment():
     if virtual_env is not None:
         return virtual_env
 
-    for environment in find_system_environments():
-        return environment
-
-    # If no Python Environment is found, use the environment we're already
+    # If no VirtualEnv is found, use the environment we're already
     # using.
     return SameEnvironment()
 
 
-@time_cache(seconds=10 * 60)  # 10 Minutes
 def get_cached_default_environment():
+    var = os.environ.get('VIRTUAL_ENV')
+    environment = _get_cached_default_environment()
+    if var and var != environment.path:
+        _get_cached_default_environment.clear_cache()
+        return _get_cached_default_environment()
+    return environment
+
+
+@time_cache(seconds=10 * 60)  # 10 Minutes
+def _get_cached_default_environment():
     return get_default_environment()
 
 
@@ -222,7 +244,7 @@ def find_virtualenvs(paths=None, **kwargs):
 
                 try:
                     executable = _get_executable_path(path, safe=safe)
-                    yield Environment(path, executable)
+                    yield Environment(executable)
                 except InvalidPythonEnvironment:
                     pass
 
@@ -246,23 +268,6 @@ def find_system_environments():
             pass
 
 
-# TODO: the logic to find the Python prefix is much more complicated than that.
-# See Modules/getpath.c for UNIX and PC/getpathp.c for Windows in CPython's
-# source code. A solution would be to deduce it by running the Python
-# interpreter and printing the value of sys.prefix.
-def _get_python_prefix(executable):
-    if os.name != 'nt':
-        return os.path.dirname(os.path.dirname(executable))
-    landmark = os.path.join('Lib', 'os.py')
-    prefix = os.path.dirname(executable)
-    while prefix:
-        if os.path.join(prefix, landmark):
-            return prefix
-        prefix = os.path.dirname(prefix)
-    raise InvalidPythonEnvironment(
-        "Cannot find prefix of executable %s." % executable)
-
-
 # TODO: this function should probably return a list of environments since
 # multiple Python installations can be found on a system for the same version.
 def get_system_environment(version):
@@ -277,17 +282,17 @@ def get_system_environment(version):
     if exe:
         if exe == sys.executable:
             return SameEnvironment()
-        return Environment(_get_python_prefix(exe), exe)
+        return Environment(exe)
 
     if os.name == 'nt':
-        for prefix, exe in _get_executables_from_windows_registry(version):
-            return Environment(prefix, exe)
+        for exe in _get_executables_from_windows_registry(version):
+            return Environment(exe)
     raise InvalidPythonEnvironment("Cannot find executable python%s." % version)
 
 
 def create_environment(path, safe=True):
     """
-    Make it possible to manually create an environment by specifying a
+    Make it possible to manually create an Environment object by specifying a
     Virtualenv path or an executable path.
 
     :raises: :exc:`.InvalidPythonEnvironment`
@@ -295,8 +300,8 @@ def create_environment(path, safe=True):
     """
     if os.path.isfile(path):
         _assert_safe(path, safe)
-        return Environment(_get_python_prefix(path), path)
-    return Environment(path, _get_executable_path(path, safe=safe))
+        return Environment(path)
+    return Environment(_get_executable_path(path, safe=safe))
 
 
 def _get_executable_path(path, safe=True):
@@ -318,16 +323,16 @@ def _get_executable_path(path, safe=True):
 def _get_executables_from_windows_registry(version):
     # The winreg module is named _winreg on Python 2.
     try:
-      import winreg
+        import winreg
     except ImportError:
-      import _winreg as winreg
+        import _winreg as winreg
 
     # TODO: support Python Anaconda.
     sub_keys = [
-      r'SOFTWARE\Python\PythonCore\{version}\InstallPath',
-      r'SOFTWARE\Wow6432Node\Python\PythonCore\{version}\InstallPath',
-      r'SOFTWARE\Python\PythonCore\{version}-32\InstallPath',
-      r'SOFTWARE\Wow6432Node\Python\PythonCore\{version}-32\InstallPath'
+        r'SOFTWARE\Python\PythonCore\{version}\InstallPath',
+        r'SOFTWARE\Wow6432Node\Python\PythonCore\{version}\InstallPath',
+        r'SOFTWARE\Python\PythonCore\{version}-32\InstallPath',
+        r'SOFTWARE\Wow6432Node\Python\PythonCore\{version}-32\InstallPath'
     ]
     for root_key in [winreg.HKEY_CURRENT_USER, winreg.HKEY_LOCAL_MACHINE]:
         for sub_key in sub_keys:
@@ -337,7 +342,7 @@ def _get_executables_from_windows_registry(version):
                     prefix = winreg.QueryValueEx(key, '')[0]
                     exe = os.path.join(prefix, 'python.exe')
                     if os.path.isfile(exe):
-                        yield prefix, exe
+                        yield exe
             except WindowsError:
                 pass
 

jedi/api/helpers.py

@@ -105,14 +105,17 @@ def get_stack_at_position(grammar, code_lines, module_node, pos):
         # TODO This is for now not an official parso API that exists purely
         #   for Jedi.
         tokens = grammar._tokenize(code)
-        for token_ in tokens:
-            if token_.string == safeword:
+        for token in tokens:
+            if token.string == safeword:
                 raise EndMarkerReached()
-            elif token_.prefix.endswith(safeword):
+            elif token.prefix.endswith(safeword):
                 # This happens with comments.
                 raise EndMarkerReached()
+            elif token.string.endswith(safeword):
+                yield token  # Probably an f-string literal that was not finished.
+                raise EndMarkerReached()
             else:
-                yield token_
+                yield token
 
     # The code might be indedented, just remove it.
     code = dedent(_get_code_for_stack(code_lines, module_node, pos))
@@ -166,7 +169,8 @@ def _get_index_and_key(nodes, position):
 
     if nodes_before:
         last = nodes_before[-1]
-        if last.type == 'argument' and last.children[1].end_pos <= position:
+        if last.type == 'argument' and last.children[1] == '=' \
+                and last.children[1].end_pos <= position:
             # Checked if the argument
             key_str = last.children[0].value
         elif last == '=':

jedi/api/project.py

@@ -85,7 +85,7 @@ class Project(object):
         if environment is None:
             environment = self.get_environment()
 
-        sys_path = environment.get_sys_path()
+        sys_path = list(environment.get_sys_path())
         try:
             sys_path.remove('')
         except ValueError:
@@ -108,12 +108,7 @@ class Project(object):
             if evaluator.script_path is not None:
                 suffixed += discover_buildout_paths(evaluator, evaluator.script_path)
 
-                traversed = []
-                for parent in traverse_parents(evaluator.script_path):
-                    traversed.append(parent)
-                    if parent == self._path:
-                        # Don't go futher than the project path.
-                        break
+                traversed = list(traverse_parents(evaluator.script_path))
 
                 # AFAIK some libraries have imports like `foo.foo.bar`, which
                 # leads to the conclusion to by default prefer longer paths

jedi/api/replstartup.py

@@ -1,6 +1,8 @@
 """
 To use Jedi completion in Python interpreter, add the following in your shell
-setup (e.g., ``.bashrc``)::
+setup (e.g., ``.bashrc``). This works only on Linux/Mac, because readline is
+not available on Windows. If you still want Jedi autocompletion in your REPL,
+just use IPython instead::
 
     export PYTHONSTARTUP="$(python -m jedi repl)"
 

jedi/cache.py

@@ -126,6 +126,7 @@ def time_cache(seconds):
 
         wrapper.clear_cache = lambda: cache.clear()
         return wrapper
+
     return decorator
 
 

jedi/debug.py

@@ -2,16 +2,17 @@ from jedi._compatibility import encoding, is_py3, u
 import os
 import time
 
+_inited = False
+
+
 def _lazy_colorama_init():
     """
-    Lazily init colorama if necessary, not to screw up stdout is debug not
-    enabled.
+    Lazily init colorama if necessary, not to screw up stdout if debugging is
+    not enabled.
 
     This version of the function does nothing.
     """
-    pass
 
-_inited=False
 
 try:
     if os.name == 'nt':
@@ -21,7 +22,8 @@ try:
         # Use colorama for nicer console output.
         from colorama import Fore, init
         from colorama import initialise
-        def _lazy_colorama_init():
+
+        def _lazy_colorama_init():  # noqa: F811
             """
             Lazily init colorama if necessary, not to screw up stdout is
             debug not enabled.

jedi/evaluate/__init__.py

@@ -105,6 +105,9 @@ class Evaluator(object):
         self.is_analysis = False
         self.project = project
         self.access_cache = {}
+        # This setting is only temporary to limit the work we have to do with
+        # tensorflow and others.
+        self.infer_enabled = True
 
         self.reset_recursion_limitations()
         self.allow_different_encoding = True
@@ -123,6 +126,9 @@ class Evaluator(object):
         return self.project._get_sys_path(self, environment=self.environment)
 
     def eval_element(self, context, element):
+        if not self.infer_enabled:
+            return NO_CONTEXTS
+
         if isinstance(context, CompForContext):
             return eval_node(context, element)
 
@@ -216,7 +222,7 @@ class Evaluator(object):
             if type_ == 'classdef':
                 return [ClassContext(self, context, name.parent)]
             elif type_ == 'funcdef':
-                return [FunctionContext(self, context, name.parent)]
+                return [FunctionContext.from_context(context, name.parent)]
 
             if type_ == 'expr_stmt':
                 is_simple_name = name.parent.type not in ('power', 'trailer')
@@ -334,16 +340,15 @@ class Evaluator(object):
             parent_context = from_scope_node(parent_scope, child_is_funcdef=is_funcdef)
 
             if is_funcdef:
+                func = FunctionContext.from_context(
+                    parent_context,
+                    scope_node
+                )
                 if isinstance(parent_context, AnonymousInstance):
                     func = BoundMethod(
-                        self, parent_context, parent_context.class_context,
-                        parent_context.parent_context, scope_node
-                    )
-                else:
-                    func = FunctionContext(
-                        self,
-                        parent_context,
-                        scope_node
+                        instance=parent_context,
+                        klass=parent_context.class_context,
+                        function=func
                     )
                 if is_nested and not node_is_object:
                     return func.get_function_execution()
@@ -373,12 +378,12 @@ class Evaluator(object):
             scope_node = parent_scope(node)
         return from_scope_node(scope_node, is_nested=True, node_is_object=node_is_object)
 
-    def parse_and_get_code(self, code=None, path=None, **kwargs):
+    def parse_and_get_code(self, code=None, path=None, encoding='utf-8', **kwargs):
         if self.allow_different_encoding:
             if code is None:
                 with open(path, 'rb') as f:
                     code = f.read()
-            code = python_bytes_to_unicode(code, errors='replace')
+            code = python_bytes_to_unicode(code, encoding=encoding, errors='replace')
 
         return self.grammar.parse(code=code, path=path, **kwargs), code
 

jedi/evaluate/arguments.py

@@ -1,3 +1,5 @@
+import re
+
 from parso.python import tree
 
 from jedi._compatibility import zip_longest
@@ -8,7 +10,7 @@ from jedi.evaluate.lazy_context import LazyKnownContext, LazyKnownContexts, \
 from jedi.evaluate.filters import ParamName
 from jedi.evaluate.base_context import NO_CONTEXTS
 from jedi.evaluate.context import iterable
-from jedi.evaluate.param import get_params, ExecutedParam
+from jedi.evaluate.param import get_executed_params, ExecutedParam
 
 
 def try_iter_content(types, depth=0):
@@ -28,32 +30,83 @@ def try_iter_content(types, depth=0):
                 try_iter_content(lazy_context.infer(), depth + 1)
 
 
+def repack_with_argument_clinic(string, keep_arguments_param=False):
+    """
+    Transforms a function or method with arguments to the signature that is
+    given as an argument clinic notation.
+
+    Argument clinic is part of CPython and used for all the functions that are
+    implemented in C (Python 3.7):
+
+        str.split.__text_signature__
+        # Results in: '($self, /, sep=None, maxsplit=-1)'
+    """
+    clinic_args = list(_parse_argument_clinic(string))
+
+    def decorator(func):
+        def wrapper(*args, **kwargs):
+            if keep_arguments_param:
+                arguments = kwargs['arguments']
+            else:
+                arguments = kwargs.pop('arguments')
+            try:
+                args += tuple(_iterate_argument_clinic(arguments, clinic_args))
+            except ValueError:
+                return NO_CONTEXTS
+            else:
+                return func(*args, **kwargs)
+
+        return wrapper
+    return decorator
+
+
+def _iterate_argument_clinic(arguments, parameters):
+    """Uses a list with argument clinic information (see PEP 436)."""
+    iterator = arguments.unpack()
+    for i, (name, optional, allow_kwargs) in enumerate(parameters):
+        key, argument = next(iterator, (None, None))
+        if key is not None:
+            debug.warning('Keyword arguments in argument clinic are currently not supported.')
+            raise ValueError
+        if argument is None and not optional:
+            debug.warning('TypeError: %s expected at least %s arguments, got %s',
+                          name, len(parameters), i)
+            raise ValueError
+
+        context_set = NO_CONTEXTS if argument is None else argument.infer()
+
+        if not context_set and not optional:
+            # For the stdlib we always want values. If we don't get them,
+            # that's ok, maybe something is too hard to resolve, however,
+            # we will not proceed with the evaluation of that function.
+            debug.warning('argument_clinic "%s" not resolvable.', name)
+            raise ValueError
+        yield context_set
+
+
+def _parse_argument_clinic(string):
+    allow_kwargs = False
+    optional = False
+    while string:
+        # Optional arguments have to begin with a bracket. And should always be
+        # at the end of the arguments. This is therefore not a proper argument
+        # clinic implementation. `range()` for exmple allows an optional start
+        # value at the beginning.
+        match = re.match('(?:(?:(\[),? ?|, ?|)(\w+)|, ?/)\]*', string)
+        string = string[len(match.group(0)):]
+        if not match.group(2):  # A slash -> allow named arguments
+            allow_kwargs = True
+            continue
+        optional = optional or bool(match.group(1))
+        word = match.group(2)
+        yield (word, optional, allow_kwargs)
+
+
 class AbstractArguments(object):
     context = None
     argument_node = None
     trailer = None
 
-    def eval_argument_clinic(self, parameters):
-        """Uses a list with argument clinic information (see PEP 436)."""
-        iterator = self.unpack()
-        for i, (name, optional, allow_kwargs) in enumerate(parameters):
-            key, argument = next(iterator, (None, None))
-            if key is not None:
-                raise NotImplementedError
-            if argument is None and not optional:
-                debug.warning('TypeError: %s expected at least %s arguments, got %s',
-                              name, len(parameters), i)
-                raise ValueError
-            values = NO_CONTEXTS if argument is None else argument.infer()
-
-            if not values and not optional:
-                # For the stdlib we always want values. If we don't get them,
-                # that's ok, maybe something is too hard to resolve, however,
-                # we will not proceed with the evaluation of that function.
-                debug.warning('argument_clinic "%s" not resolvable.', name)
-                raise ValueError
-            yield values
-
     def eval_all(self, funcdef=None):
         """
         Evaluates all arguments as a support for static analysis
@@ -64,17 +117,17 @@ class AbstractArguments(object):
             try_iter_content(types)
 
     def get_calling_nodes(self):
-        raise NotImplementedError
+        return []
 
     def unpack(self, funcdef=None):
         raise NotImplementedError
 
-    def get_params(self, execution_context):
-        return get_params(execution_context, self)
+    def get_executed_params(self, execution_context):
+        return get_executed_params(execution_context, self)
 
 
 class AnonymousArguments(AbstractArguments):
-    def get_params(self, execution_context):
+    def get_executed_params(self, execution_context):
         from jedi.evaluate.dynamic import search_params
         return search_params(
             execution_context.evaluator,
@@ -82,6 +135,9 @@ class AnonymousArguments(AbstractArguments):
             execution_context.tree_node
         )
 
+    def __repr__(self):
+        return '%s()' % self.__class__.__name__
+
 
 class TreeArguments(AbstractArguments):
     def __init__(self, evaluator, context, argument_node, trailer=None):
@@ -171,7 +227,7 @@ class TreeArguments(AbstractArguments):
         return '<%s: %s>' % (self.__class__.__name__, self.argument_node)
 
     def get_calling_nodes(self):
-        from jedi.evaluate.dynamic import MergedExecutedParams
+        from jedi.evaluate.dynamic import DynamicExecutedParams
         old_arguments_list = []
         arguments = self
 
@@ -190,7 +246,7 @@ class TreeArguments(AbstractArguments):
                 if not isinstance(names[0], ParamName):
                     break
                 param = names[0].get_param()
-                if isinstance(param, MergedExecutedParams):
+                if isinstance(param, DynamicExecutedParams):
                     # For dynamic searches we don't even want to see errors.
                     return []
                 if not isinstance(param, ExecutedParam):
@@ -215,9 +271,6 @@ class ValuesArguments(AbstractArguments):
         for values in self._values_list:
             yield None, LazyKnownContexts(values)
 
-    def get_calling_nodes(self):
-        return []
-
     def __repr__(self):
         return '<%s: %s>' % (self.__class__.__name__, self._values_list)
 

jedi/evaluate/base_context.py

@@ -199,9 +199,10 @@ def iterate_contexts(contexts, contextualized_node=None, is_async=False):
 
 
 class TreeContext(Context):
-    def __init__(self, evaluator, parent_context=None):
+    def __init__(self, evaluator, parent_context, tree_node):
         super(TreeContext, self).__init__(evaluator, parent_context)
         self.predefined_names = {}
+        self.tree_node = tree_node
 
     def __repr__(self):
         return '<%s: %s>' % (self.__class__.__name__, self.tree_node)

jedi/evaluate/compiled/__init__.py

@@ -32,8 +32,12 @@ def get_string_context_set(evaluator):
     return builtin_from_name(evaluator, u'str').execute_evaluated()
 
 
-def load_module(evaluator, **kwargs):
-    access_path = evaluator.compiled_subprocess.load_module(**kwargs)
+def load_module(evaluator, dotted_name, **kwargs):
+    # Temporary, some tensorflow builtins cannot be loaded, so it's tried again
+    # and again and it's really slow.
+    if dotted_name.startswith('tensorflow.'):
+        return None
+    access_path = evaluator.compiled_subprocess.load_module(dotted_name=dotted_name, **kwargs)
     if access_path is None:
         return None
     return create_from_access_path(evaluator, access_path)

jedi/evaluate/compiled/access.py

@@ -5,11 +5,9 @@ from textwrap import dedent
 import operator as op
 from collections import namedtuple
 
-from jedi import debug
-from jedi._compatibility import unicode, is_py3, is_py34, builtins, \
+from jedi._compatibility import unicode, is_py3, builtins, \
     py_version, force_unicode, print_to_stderr
 from jedi.evaluate.compiled.getattr_static import getattr_static
-from jedi.evaluate.utils import dotted_from_fs_path
 
 
 MethodDescriptorType = type(str.replace)
@@ -33,10 +31,9 @@ NOT_CLASS_TYPES = (
 if is_py3:
     NOT_CLASS_TYPES += (
         types.MappingProxyType,
-        types.SimpleNamespace
+        types.SimpleNamespace,
+        types.DynamicClassAttribute,
     )
-    if is_py34:
-        NOT_CLASS_TYPES += (types.DynamicClassAttribute,)
 
 
 # Those types don't exist in typing.
@@ -137,20 +134,13 @@ def create_access(evaluator, obj):
     return evaluator.compiled_subprocess.get_or_create_access_handle(obj)
 
 
-def load_module(evaluator, path=None, name=None, sys_path=None):
-    if sys_path is None:
-        sys_path = list(evaluator.get_sys_path())
-    if path is not None:
-        dotted_path = dotted_from_fs_path(path, sys_path=sys_path)
-    else:
-        dotted_path = name
-
+def load_module(evaluator, dotted_name, sys_path):
     temp, sys.path = sys.path, sys_path
     try:
-        __import__(dotted_path)
+        __import__(dotted_name)
     except ImportError:
         # If a module is "corrupt" or not really a Python module or whatever.
-        debug.warning('Module %s not importable in path %s.', dotted_path, path)
+        print_to_stderr('Module %s not importable in path %s.' % (dotted_name, sys_path))
         return None
     except Exception:
         # Since __import__ pretty much makes code execution possible, just
@@ -163,7 +153,7 @@ def load_module(evaluator, path=None, name=None, sys_path=None):
 
     # Just access the cache after import, because of #59 as well as the very
     # complicated import structure of Python.
-    module = sys.modules[dotted_path]
+    module = sys.modules[dotted_name]
     return create_access_path(evaluator, module)
 
 
@@ -262,6 +252,9 @@ class DirectObjectAccess(object):
     def py__bases__(self):
         return [self._create_access_path(base) for base in self._obj.__bases__]
 
+    def py__path__(self):
+        return self._obj.__path__
+
     @_force_unicode_decorator
     def get_repr(self):
         builtins = 'builtins', '__builtin__'
@@ -476,7 +469,7 @@ else:
 
 class _SPECIAL_OBJECTS(object):
     FUNCTION_CLASS = types.FunctionType
-    METHOD_CLASS = type(DirectObjectAccess.py__bool__)
+    BOUND_METHOD_CLASS = type(DirectObjectAccess(None, None).py__bool__)
     MODULE_CLASS = types.ModuleType
     GENERATOR_OBJECT = _a_generator(1.0)
     BUILTINS = builtins

jedi/evaluate/compiled/context.py

@@ -55,7 +55,7 @@ class CompiledObject(Context):
             return FunctionContext(
                 self.evaluator,
                 parent_context=self.parent_context,
-                funcdef=self.tree_node
+                tree_node=self.tree_node
             ).py__call__(params)
         if self.access_handle.is_class():
             from jedi.evaluate.context import CompiledInstance
@@ -81,6 +81,10 @@ class CompiledObject(Context):
             for access in self.access_handle.py__bases__()
         )
 
+    @CheckAttribute
+    def py__path__(self):
+        return self.access_handle.py__path__()
+
     def py__bool__(self):
         return self.access_handle.py__bool__()
 
@@ -443,8 +447,11 @@ def create_from_name(evaluator, compiled_object, name):
         pass
 
     access = compiled_object.access_handle.getattr(name, default=None)
+    parent_context = compiled_object
+    if parent_context.is_class():
+        parent_context = parent_context.parent_context
     return create_cached_compiled_object(
-        evaluator, access, parent_context=compiled_object, faked=faked
+        evaluator, access, parent_context=parent_context, faked=faked
     )
 
 

jedi/evaluate/compiled/mixed.py

@@ -107,7 +107,7 @@ def _load_module(evaluator, path):
     module_node = evaluator.grammar.parse(
         path=path,
         cache=True,
-        diff_cache=True,
+        diff_cache=settings.fast_parser,
         cache_path=settings.cache_directory
     ).get_root_node()
     # python_module = inspect.getmodule(python_object)

jedi/evaluate/compiled/subprocess/__init__.py

@@ -15,27 +15,41 @@ import errno
 import weakref
 import traceback
 from functools import partial
+from threading import Thread
+try:
+    from queue import Queue, Empty
+except ImportError:
+    from Queue import Queue, Empty  # python 2.7
 
 from jedi._compatibility import queue, is_py3, force_unicode, \
-    pickle_dump, pickle_load, highest_pickle_protocol, GeneralizedPopen
+    pickle_dump, pickle_load, GeneralizedPopen, print_to_stderr
+from jedi import debug
 from jedi.cache import memoize_method
 from jedi.evaluate.compiled.subprocess import functions
 from jedi.evaluate.compiled.access import DirectObjectAccess, AccessPath, \
     SignatureParam
 from jedi.api.exceptions import InternalError
 
-_subprocesses = {}
 
 _MAIN_PATH = os.path.join(os.path.dirname(__file__), '__main__.py')
 
 
-def get_subprocess(executable, version):
-    try:
-        return _subprocesses[executable]
-    except KeyError:
-        sub = _subprocesses[executable] = _CompiledSubprocess(executable,
-                                                              version)
-        return sub
+def _enqueue_output(out, queue):
+    for line in iter(out.readline, b''):
+        queue.put(line)
+    out.close()
+
+
+def _add_stderr_to_debug(stderr_queue):
+    while True:
+        # Try to do some error reporting from the subprocess and print its
+        # stderr contents.
+        try:
+            line = stderr_queue.get_nowait()
+            line = line.decode('utf-8', 'replace')
+            debug.warning('stderr output: %s' % line.rstrip('\n'))
+        except Empty:
+            break
 
 
 def _get_function(name):
@@ -119,30 +133,41 @@ class EvaluatorSubprocess(_EvaluatorProcess):
         return obj
 
     def __del__(self):
-        if self._used:
+        if self._used and not self._compiled_subprocess.is_crashed:
             self._compiled_subprocess.delete_evaluator(self._evaluator_id)
 
 
-class _CompiledSubprocess(object):
-    _crashed = False
+class CompiledSubprocess(object):
+    is_crashed = False
+    # Start with 2, gets set after _get_info.
+    _pickle_protocol = 2
 
-    def __init__(self, executable, version):
+    def __init__(self, executable):
         self._executable = executable
         self._evaluator_deletion_queue = queue.deque()
-        self._pickle_protocol = highest_pickle_protocol([sys.version_info,
-                                                         version])
+
+    def __repr__(self):
+        pid = os.getpid()
+        return '<%s _executable=%r, _pickle_protocol=%r, is_crashed=%r, pid=%r>' % (
+            self.__class__.__name__,
+            self._executable,
+            self._pickle_protocol,
+            self.is_crashed,
+            pid,
+        )
 
     @property
     @memoize_method
     def _process(self):
+        debug.dbg('Start environment subprocess %s', self._executable)
         parso_path = sys.modules['parso'].__file__
         args = (
             self._executable,
             _MAIN_PATH,
             os.path.dirname(os.path.dirname(parso_path)),
-            str(self._pickle_protocol)
+            '.'.join(str(x) for x in sys.version_info[:3]),
         )
-        return GeneralizedPopen(
+        process = GeneralizedPopen(
             args,
             stdin=subprocess.PIPE,
             stdout=subprocess.PIPE,
@@ -151,6 +176,14 @@ class _CompiledSubprocess(object):
             # (this is already the case on Python 3).
             bufsize=-1
         )
+        self._stderr_queue = Queue()
+        self._stderr_thread = t = Thread(
+            target=_enqueue_output,
+            args=(process.stderr, self._stderr_queue)
+        )
+        t.daemon = True
+        t.start()
+        return process
 
     def run(self, evaluator, function, args=(), kwargs={}):
         # Delete old evaluators.
@@ -168,24 +201,23 @@ class _CompiledSubprocess(object):
     def get_sys_path(self):
         return self._send(None, functions.get_sys_path, (), {})
 
-    def kill(self):
-        self._crashed = True
+    def _kill(self):
+        self.is_crashed = True
         try:
-            subprocess = _subprocesses[self._executable]
-        except KeyError:
-            # Fine it was already removed from the cache.
+            self._process.kill()
+            self._process.wait()
+        except (AttributeError, TypeError):
+            # If the Python process is terminating, it will remove some modules
+            # earlier than others and in general it's unclear how to deal with
+            # that so we just ignore the exceptions here.
             pass
-        else:
-            # In the `!=` case there is already a new subprocess in place
-            # and we don't need to do anything here anymore.
-            if subprocess == self:
-                del _subprocesses[self._executable]
 
-        self._process.kill()
-        self._process.wait()
+    def __del__(self):
+        if not self.is_crashed:
+            self._kill()
 
     def _send(self, evaluator_id, function, args=(), kwargs={}):
-        if self._crashed:
+        if self.is_crashed:
             raise InternalError("The subprocess %s has crashed." % self._executable)
 
         if not is_py3:
@@ -202,7 +234,7 @@ class _CompiledSubprocess(object):
             if e.errno not in (errno.EPIPE, errno.EINVAL):
                 # Not a broken pipe
                 raise
-            self.kill()
+            self._kill()
             raise InternalError("The subprocess %s was killed. Maybe out of memory?"
                                 % self._executable)
 
@@ -210,10 +242,11 @@ class _CompiledSubprocess(object):
             is_exception, traceback, result = pickle_load(self._process.stdout)
         except EOFError as eof_error:
             try:
-                stderr = self._process.stderr.read()
+                stderr = self._process.stderr.read().decode('utf-8', 'replace')
             except Exception as exc:
                 stderr = '<empty/not available (%r)>' % exc
-            self.kill()
+            self._kill()
+            _add_stderr_to_debug(self._stderr_queue)
             raise InternalError(
                 "The subprocess %s has crashed (%r, stderr=%s)." % (
                     self._executable,
@@ -221,6 +254,8 @@ class _CompiledSubprocess(object):
                     stderr,
                 ))
 
+        _add_stderr_to_debug(self._stderr_queue)
+
         if is_exception:
             # Replace the attribute error message with a the traceback. It's
             # way more informative.
@@ -284,11 +319,9 @@ class Listener(object):
 
     def listen(self):
         stdout = sys.stdout
-        # Mute stdout/stderr. Nobody should actually be able to write to those,
-        # because stdout is used for IPC and stderr will just be annoying if it
-        # leaks (on module imports).
+        # Mute stdout. Nobody should actually be able to write to it,
+        # because stdout is used for IPC.
         sys.stdout = open(os.devnull, 'w')
-        sys.stderr = open(os.devnull, 'w')
         stdin = sys.stdin
         if sys.version_info[0] > 2:
             stdout = stdout.buffer
@@ -304,9 +337,9 @@ class Listener(object):
             try:
                 payload = pickle_load(stdin)
             except EOFError:
-                # It looks like the parent process closed. Don't make a big fuss
-                # here and just exit.
-                exit(1)
+                # It looks like the parent process closed.
+                # Don't make a big fuss here and just exit.
+                exit(0)
             try:
                 result = False, None, self._run(*payload)
             except Exception as e:

jedi/evaluate/compiled/subprocess/__main__.py

@@ -1,5 +1,5 @@
-import sys
 import os
+import sys
 
 
 def _get_paths():
@@ -45,7 +45,11 @@ else:
     load('jedi')
     from jedi.evaluate.compiled import subprocess  # NOQA
 
+from jedi._compatibility import highest_pickle_protocol  # noqa: E402
+
+
 # Retrieve the pickle protocol.
-pickle_protocol = int(sys.argv[2])
+host_sys_version = [int(x) for x in sys.argv[2].split('.')]
+pickle_protocol = highest_pickle_protocol([sys.version_info, host_sys_version])
 # And finally start the client.
-subprocess.Listener(pickle_protocol).listen()
+subprocess.Listener(pickle_protocol=pickle_protocol).listen()

jedi/evaluate/context/function.py

@@ -38,17 +38,9 @@ class LambdaName(AbstractNameDefinition):
         return ContextSet(self._lambda_context)
 
 
-class FunctionContext(use_metaclass(CachedMetaClass, TreeContext)):
-    """
-    Needed because of decorators. Decorators are evaluated here.
-    """
+class AbstractFunction(TreeContext):
     api_type = u'function'
 
-    def __init__(self, evaluator, parent_context, funcdef):
-        """ This should not be called directly """
-        super(FunctionContext, self).__init__(evaluator, parent_context)
-        self.tree_node = funcdef
-
     def get_filters(self, search_global, until_position=None, origin_scope=None):
         if search_global:
             yield ParserTreeFilter(
@@ -62,6 +54,24 @@ class FunctionContext(use_metaclass(CachedMetaClass, TreeContext)):
             for filter in scope.get_filters(search_global=False, origin_scope=origin_scope):
                 yield filter
 
+    def get_param_names(self):
+        function_execution = self.get_function_execution()
+        return [ParamName(function_execution, param.name)
+                for param in self.tree_node.get_params()]
+
+    @property
+    def name(self):
+        if self.tree_node.type == 'lambdef':
+            return LambdaName(self)
+        return ContextName(self, self.tree_node.name)
+
+    def get_function_execution(self, arguments=None):
+        raise NotImplementedError
+
+    def py__call__(self, arguments):
+        function_execution = self.get_function_execution(arguments)
+        return self.infer_function_execution(function_execution)
+
     def infer_function_execution(self, function_execution):
         """
         Created to be used by inheritance.
@@ -84,35 +94,31 @@ class FunctionContext(use_metaclass(CachedMetaClass, TreeContext)):
             else:
                 return function_execution.get_return_values()
 
+    def py__name__(self):
+        return self.name.string_name
+
+
+class FunctionContext(use_metaclass(CachedMetaClass, AbstractFunction)):
+    """
+    Needed because of decorators. Decorators are evaluated here.
+    """
+    @classmethod
+    def from_context(cls, context, tree_node):
+        from jedi.evaluate.context import AbstractInstanceContext
+
+        while context.is_class() or isinstance(context, AbstractInstanceContext):
+            context = context.parent_context
+
+        return cls(context.evaluator, parent_context=context, tree_node=tree_node)
+
     def get_function_execution(self, arguments=None):
         if arguments is None:
             arguments = AnonymousArguments()
 
         return FunctionExecutionContext(self.evaluator, self.parent_context, self, arguments)
 
-    def py__call__(self, arguments):
-        function_execution = self.get_function_execution(arguments)
-        return self.infer_function_execution(function_execution)
-
     def py__class__(self):
-        # This differentiation is only necessary for Python2. Python3 does not
-        # use a different method class.
-        if isinstance(parser_utils.get_parent_scope(self.tree_node), tree.Class):
-            name = u'METHOD_CLASS'
-        else:
-            name = u'FUNCTION_CLASS'
-        return compiled.get_special_object(self.evaluator, name)
-
-    @property
-    def name(self):
-        if self.tree_node.type == 'lambdef':
-            return LambdaName(self)
-        return ContextName(self, self.tree_node.name)
-
-    def get_param_names(self):
-        function_execution = self.get_function_execution()
-        return [ParamName(function_execution, param.name)
-                for param in self.tree_node.get_params()]
+        return compiled.get_special_object(self.evaluator, u'FUNCTION_CLASS')
 
 
 class FunctionExecutionContext(TreeContext):
@@ -127,9 +133,12 @@ class FunctionExecutionContext(TreeContext):
     function_execution_filter = FunctionExecutionFilter
 
     def __init__(self, evaluator, parent_context, function_context, var_args):
-        super(FunctionExecutionContext, self).__init__(evaluator, parent_context)
+        super(FunctionExecutionContext, self).__init__(
+            evaluator,
+            parent_context,
+            function_context.tree_node,
+        )
         self.function_context = function_context
-        self.tree_node = function_context.tree_node
         self.var_args = var_args
 
     @evaluator_method_cache(default=NO_CONTEXTS)
@@ -240,5 +249,5 @@ class FunctionExecutionContext(TreeContext):
                                              origin_scope=origin_scope)
 
     @evaluator_method_cache()
-    def get_params(self):
-        return self.var_args.get_params(self)
+    def get_executed_params(self):
+        return self.var_args.get_executed_params(self)

jedi/evaluate/context/instance.py

@@ -1,6 +1,7 @@
 from abc import abstractproperty
 
 from jedi import debug
+from jedi import settings
 from jedi.evaluate import compiled
 from jedi.evaluate import filters
 from jedi.evaluate.base_context import Context, NO_CONTEXTS, ContextSet, \
@@ -8,34 +9,40 @@ from jedi.evaluate.base_context import Context, NO_CONTEXTS, ContextSet, \
 from jedi.evaluate.lazy_context import LazyKnownContext, LazyKnownContexts
 from jedi.evaluate.cache import evaluator_method_cache
 from jedi.evaluate.arguments import AbstractArguments, AnonymousArguments
-from jedi.cache import memoize_method
-from jedi.evaluate.context.function import FunctionExecutionContext, FunctionContext
-from jedi.evaluate.context.klass import ClassContext, apply_py__get__
+from jedi.evaluate.context.function import FunctionExecutionContext, \
+    FunctionContext, AbstractFunction
+from jedi.evaluate.context.klass import ClassContext, apply_py__get__, ClassFilter
 from jedi.evaluate.context import iterable
 from jedi.parser_utils import get_parent_scope
 
 
-class BaseInstanceFunctionExecution(FunctionExecutionContext):
-    def __init__(self, instance, *args, **kwargs):
-        self.instance = instance
-        super(BaseInstanceFunctionExecution, self).__init__(
-            instance.evaluator, *args, **kwargs)
+class InstanceExecutedParam(object):
+    def __init__(self, instance):
+        self._instance = instance
+
+    def infer(self):
+        return ContextSet(self._instance)
 
 
-class InstanceFunctionExecution(BaseInstanceFunctionExecution):
-    def __init__(self, instance, parent_context, function_context, var_args):
-        var_args = InstanceVarArgs(self, var_args)
+class AnonymousInstanceArguments(AnonymousArguments):
+    def __init__(self, instance):
+        self._instance = instance
 
-        super(InstanceFunctionExecution, self).__init__(
-            instance, parent_context, function_context, var_args)
-
-
-class AnonymousInstanceFunctionExecution(BaseInstanceFunctionExecution):
-    function_execution_filter = filters.AnonymousInstanceFunctionExecutionFilter
-
-    def __init__(self, instance, parent_context, function_context, var_args):
-        super(AnonymousInstanceFunctionExecution, self).__init__(
-            instance, parent_context, function_context, var_args)
+    def get_executed_params(self, execution_context):
+        from jedi.evaluate.dynamic import search_params
+        self_param = InstanceExecutedParam(self._instance)
+        tree_params = execution_context.tree_node.get_params()
+        if len(tree_params) == 1:
+            # If the only param is self, we don't need to try to find
+            # executions of this function, we have all the params already.
+            return [self_param]
+        executed_params = list(search_params(
+            execution_context.evaluator,
+            execution_context,
+            execution_context.tree_node
+        ))
+        executed_params[0] = self_param
+        return executed_params
 
 
 class AbstractInstanceContext(Context):
@@ -43,7 +50,6 @@ class AbstractInstanceContext(Context):
     This class is used to evaluate instances.
     """
     api_type = u'instance'
-    function_execution_cls = InstanceFunctionExecution
 
     def __init__(self, evaluator, parent_context, class_context, var_args):
         super(AbstractInstanceContext, self).__init__(evaluator, parent_context)
@@ -63,7 +69,7 @@ class AbstractInstanceContext(Context):
             raise AttributeError
 
         def execute(arguments):
-            return ContextSet.from_sets(name.execute(arguments) for name in names)
+            return ContextSet.from_sets(name.infer().execute(arguments) for name in names)
 
         return execute
 
@@ -86,7 +92,7 @@ class AbstractInstanceContext(Context):
 
     def execute_function_slots(self, names, *evaluated_args):
         return ContextSet.from_sets(
-            name.execute_evaluated(*evaluated_args)
+            name.infer().execute_evaluated(*evaluated_args)
             for name in names
         )
 
@@ -158,21 +164,18 @@ class AbstractInstanceContext(Context):
     def name(self):
         pass
 
-    def _create_init_execution(self, class_context, func_node):
-        bound_method = BoundMethod(
-            self.evaluator, self, class_context, self.parent_context, func_node
-        )
-        return self.function_execution_cls(
-            self,
-            class_context.parent_context,
-            bound_method,
-            self.var_args
-        )
+    def _create_init_execution(self, class_context, bound_method):
+        return bound_method.get_function_execution(self.var_args)
 
     def create_init_executions(self):
         for name in self.get_function_slot_names(u'__init__'):
-            if isinstance(name, SelfName):
-                yield self._create_init_execution(name.class_context, name.tree_name.parent)
+            if isinstance(name, LazyInstanceClassName):
+                function = FunctionContext.from_context(
+                    self.parent_context,
+                    name.tree_name.parent
+                )
+                bound_method = BoundMethod(self, name.class_context, function)
+                yield self._create_init_execution(name.class_context, bound_method)
 
     @evaluator_method_cache()
     def create_instance_context(self, class_context, node):
@@ -184,13 +187,14 @@ class AbstractInstanceContext(Context):
         else:
             parent_context = self.create_instance_context(class_context, scope)
             if scope.type == 'funcdef':
+                func = FunctionContext.from_context(
+                    parent_context,
+                    scope,
+                )
+                bound_method = BoundMethod(self, class_context, func)
                 if scope.name.value == '__init__' and parent_context == class_context:
-                    return self._create_init_execution(class_context, scope)
+                    return self._create_init_execution(class_context, bound_method)
                 else:
-                    bound_method = BoundMethod(
-                        self.evaluator, self, class_context,
-                        parent_context, scope
-                    )
                     return bound_method.get_function_execution()
             elif scope.type == 'classdef':
                 class_context = ClassContext(self.evaluator, parent_context, scope)
@@ -208,16 +212,18 @@ class AbstractInstanceContext(Context):
 
 
 class CompiledInstance(AbstractInstanceContext):
-    def __init__(self, *args, **kwargs):
-        super(CompiledInstance, self).__init__(*args, **kwargs)
+    def __init__(self, evaluator, parent_context, class_context, var_args):
+        self._original_var_args = var_args
+
         # I don't think that dynamic append lookups should happen here. That
         # sounds more like something that should go to py__iter__.
-        self._original_var_args = self.var_args
-
-        if self.class_context.name.string_name in ['list', 'set'] \
-                and self.parent_context.get_root_context() == self.evaluator.builtins_module:
+        if class_context.py__name__() in ['list', 'set'] \
+                and parent_context.get_root_context() == evaluator.builtins_module:
             # compare the module path with the builtin name.
-            self.var_args = iterable.get_dynamic_array_instance(self)
+            if settings.dynamic_array_additions:
+                var_args = iterable.get_dynamic_array_instance(self, var_args)
+
+        super(CompiledInstance, self).__init__(evaluator, parent_context, class_context, var_args)
 
     @property
     def name(self):
@@ -249,35 +255,33 @@ class TreeInstance(AbstractInstanceContext):
 
 
 class AnonymousInstance(TreeInstance):
-    function_execution_cls = AnonymousInstanceFunctionExecution
-
     def __init__(self, evaluator, parent_context, class_context):
         super(AnonymousInstance, self).__init__(
             evaluator,
             parent_context,
             class_context,
-            var_args=AnonymousArguments(),
+            var_args=AnonymousInstanceArguments(self),
         )
 
 
 class CompiledInstanceName(compiled.CompiledName):
-    def __init__(self, evaluator, instance, parent_context, name):
-        super(CompiledInstanceName, self).__init__(evaluator, parent_context, name)
+
+    def __init__(self, evaluator, instance, klass, name):
+        super(CompiledInstanceName, self).__init__(
+            evaluator,
+            klass.parent_context,
+            name.string_name
+        )
         self._instance = instance
+        self._class = klass
+        self._class_member_name = name
 
     @iterator_to_context_set
     def infer(self):
-        for result_context in super(CompiledInstanceName, self).infer():
+        for result_context in self._class_member_name.infer():
             is_function = result_context.api_type == 'function'
             if result_context.tree_node is not None and is_function:
-                parent_context = result_context.parent_context
-                while parent_context.is_class():
-                    parent_context = parent_context.parent_context
-
-                yield BoundMethod(
-                    result_context.evaluator, self._instance, self.parent_context,
-                    parent_context, result_context.tree_node
-                )
+                yield BoundMethod(self._instance, self._class, result_context)
             else:
                 if is_function:
                     yield CompiledBoundMethod(result_context)
@@ -285,36 +289,59 @@ class CompiledInstanceName(compiled.CompiledName):
                     yield result_context
 
 
-class CompiledInstanceClassFilter(compiled.CompiledObjectFilter):
+class CompiledInstanceClassFilter(filters.AbstractFilter):
     name_class = CompiledInstanceName
 
-    def __init__(self, evaluator, instance, compiled_object):
-        super(CompiledInstanceClassFilter, self).__init__(
-            evaluator,
-            compiled_object,
-            is_instance=True,
+    def __init__(self, evaluator, instance, klass):
+        self._evaluator = evaluator
+        self._instance = instance
+        self._class = klass
+        self._class_filter = next(klass.get_filters(is_instance=True))
+
+    def get(self, name):
+        return self._convert(self._class_filter.get(name))
+
+    def values(self):
+        return self._convert(self._class_filter.values())
+
+    def _convert(self, names):
+        return [
+            CompiledInstanceName(self._evaluator, self._instance, self._class, n)
+            for n in names
+        ]
+
+
+class BoundMethod(AbstractFunction):
+    def __init__(self, instance, klass, function):
+        super(BoundMethod, self).__init__(
+            function.evaluator,
+            function.parent_context,
+            function.tree_node,
         )
         self._instance = instance
+        self._class = klass
+        self._function = function
 
-    def _create_name(self, name):
-        return self.name_class(
-            self._evaluator, self._instance, self._compiled_object, name)
-
-
-class BoundMethod(FunctionContext):
-    def __init__(self, evaluator, instance, class_context, *args, **kwargs):
-        super(BoundMethod, self).__init__(evaluator, *args, **kwargs)
-        self._instance = instance
-        self._class_context = class_context
+    def py__class__(self):
+        return compiled.get_special_object(self.evaluator, u'BOUND_METHOD_CLASS')
 
     def get_function_execution(self, arguments=None):
         if arguments is None:
-            arguments = AnonymousArguments()
-            return AnonymousInstanceFunctionExecution(
-                self._instance, self.parent_context, self, arguments)
-        else:
-            return InstanceFunctionExecution(
-                self._instance, self.parent_context, self, arguments)
+            arguments = AnonymousInstanceArguments(self._instance)
+
+        arguments = InstanceArguments(self._instance, arguments)
+
+        if isinstance(self._function, compiled.CompiledObject):
+            # This is kind of weird, because it's coming from a compiled object
+            # and we're not sure if we want that in the future.
+            return FunctionExecutionContext(
+                self.evaluator, self.parent_context, self, arguments
+            )
+
+        return self._function.get_function_execution(arguments)
+
+    def __repr__(self):
+        return '<%s: %s>' % (self.__class__.__name__, self._function)
 
 
 class CompiledBoundMethod(compiled.CompiledObject):
@@ -326,11 +353,6 @@ class CompiledBoundMethod(compiled.CompiledObject):
         return list(super(CompiledBoundMethod, self).get_param_names())[1:]
 
 
-class InstanceNameDefinition(filters.TreeNameDefinition):
-    def infer(self):
-        return super(InstanceNameDefinition, self).infer()
-
-
 class SelfName(filters.TreeNameDefinition):
     """
     This name calculates the parent_context lazily.
@@ -345,65 +367,69 @@ class SelfName(filters.TreeNameDefinition):
         return self._instance.create_instance_context(self.class_context, self.tree_name)
 
 
-class LazyInstanceClassName(SelfName):
+class LazyInstanceClassName(object):
+    def __init__(self, instance, class_context, class_member_name):
+        self._instance = instance
+        self.class_context = class_context
+        self._class_member_name = class_member_name
+
     @iterator_to_context_set
     def infer(self):
-        for result_context in super(LazyInstanceClassName, self).infer():
+        for result_context in self._class_member_name.infer():
             if isinstance(result_context, FunctionContext):
                 # Classes are never used to resolve anything within the
                 # functions. Only other functions and modules will resolve
                 # those things.
-                parent_context = result_context.parent_context
-                while parent_context.is_class():
-                    parent_context = parent_context.parent_context
-
-                yield BoundMethod(
-                    result_context.evaluator, self._instance, self.class_context,
-                    parent_context, result_context.tree_node
-                )
+                yield BoundMethod(self._instance, self.class_context, result_context)
             else:
                 for c in apply_py__get__(result_context, self._instance):
                     yield c
 
+    def __getattr__(self, name):
+        return getattr(self._class_member_name, name)
 
-class InstanceClassFilter(filters.ParserTreeFilter):
-    name_class = LazyInstanceClassName
 
+class InstanceClassFilter(filters.AbstractFilter):
+    """
+    This filter is special in that it uses the class filter and wraps the
+    resulting names in LazyINstanceClassName. The idea is that the class name
+    filtering can be very flexible and always be reflected in instances.
+    """
     def __init__(self, evaluator, context, class_context, origin_scope):
-        super(InstanceClassFilter, self).__init__(
-            evaluator=evaluator,
-            context=context,
-            node_context=class_context,
-            origin_scope=origin_scope
-        )
+        self._instance = context
         self._class_context = class_context
+        self._class_filter = next(class_context.get_filters(
+            search_global=False,
+            origin_scope=origin_scope,
+            is_instance=True,
+        ))
 
-    def _equals_origin_scope(self):
-        node = self._origin_scope
-        while node is not None:
-            if node == self._parser_scope or node == self.context:
-                return True
-            node = get_parent_scope(node)
-        return False
+    def get(self, name):
+        return self._convert(self._class_filter.get(name))
 
-    def _access_possible(self, name):
-        return not name.value.startswith('__') or name.value.endswith('__') \
-            or self._equals_origin_scope()
+    def values(self):
+        return self._convert(self._class_filter.values())
 
-    def _filter(self, names):
-        names = super(InstanceClassFilter, self)._filter(names)
-        return [name for name in names if self._access_possible(name)]
-
-    def _convert_names(self, names):
-        return [self.name_class(self.context, self._class_context, name) for name in names]
+    def _convert(self, names):
+        return [LazyInstanceClassName(self._instance, self._class_context, n) for n in names]
 
 
-class SelfAttributeFilter(InstanceClassFilter):
+class SelfAttributeFilter(ClassFilter):
     """
     This class basically filters all the use cases where `self.*` was assigned.
     """
     name_class = SelfName
 
+    def __init__(self, evaluator, context, class_context, origin_scope):
+        super(SelfAttributeFilter, self).__init__(
+            evaluator=evaluator,
+            context=context,
+            node_context=class_context,
+            origin_scope=origin_scope,
+            is_instance=True,
+        )
+        self._class_context = class_context
+
     def _filter(self, names):
         names = self._filter_self_names(names)
         if isinstance(self._parser_scope, compiled.CompiledObject) and False:
@@ -422,19 +448,18 @@ class SelfAttributeFilter(InstanceClassFilter):
                 if name.is_definition() and self._access_possible(name):
                     yield name
 
+    def _convert_names(self, names):
+        return [self.name_class(self.context, self._class_context, name) for name in names]
+
     def _check_flows(self, names):
         return names
 
 
-class InstanceVarArgs(AbstractArguments):
-    def __init__(self, execution_context, var_args):
-        self._execution_context = execution_context
+class InstanceArguments(AbstractArguments):
+    def __init__(self, instance, var_args):
+        self.instance = instance
         self._var_args = var_args
 
-    @memoize_method
-    def _get_var_args(self):
-        return self._var_args
-
     @property
     def argument_node(self):
         return self._var_args.argument_node
@@ -444,9 +469,15 @@ class InstanceVarArgs(AbstractArguments):
         return self._var_args.trailer
 
     def unpack(self, func=None):
-        yield None, LazyKnownContext(self._execution_context.instance)
-        for values in self._get_var_args().unpack(func):
+        yield None, LazyKnownContext(self.instance)
+        for values in self._var_args.unpack(func):
             yield values
 
     def get_calling_nodes(self):
-        return self._get_var_args().get_calling_nodes()
+        return self._var_args.get_calling_nodes()
+
+    def get_executed_params(self, execution_context):
+        if isinstance(self._var_args, AnonymousInstanceArguments):
+            return self._var_args.get_executed_params(execution_context)
+
+        return super(InstanceArguments, self).get_executed_params(execution_context)

jedi/evaluate/context/iterable.py

@@ -83,10 +83,6 @@ class CompForContext(TreeContext):
     def from_comp_for(cls, parent_context, comp_for):
         return cls(parent_context.evaluator, parent_context, comp_for)
 
-    def __init__(self, evaluator, parent_context, comp_for):
-        super(CompForContext, self).__init__(evaluator, parent_context)
-        self.tree_node = comp_for
-
     def get_node(self):
         return self.tree_node
 
@@ -271,6 +267,11 @@ class DictComprehension(ComprehensionMixin, Sequence):
 
         return ContextSet(FakeSequence(self.evaluator, u'list', lazy_contexts))
 
+    def exact_key_items(self):
+        # NOTE: A smarter thing can probably done here to achieve better
+        # completions, but at least like this jedi doesn't crash
+        return []
+
 
 class GeneratorComprehension(ComprehensionMixin, GeneratorBase):
     pass
@@ -350,21 +351,39 @@ class SequenceLiteralContext(Sequence):
             return []  # Direct closing bracket, doesn't contain items.
 
         if array_node.type == 'testlist_comp':
-            return array_node.children[::2]
+            # filter out (for now) pep 448 single-star unpacking
+            return [value for value in array_node.children[::2]
+                    if value.type != "star_expr"]
         elif array_node.type == 'dictorsetmaker':
             kv = []
             iterator = iter(array_node.children)
             for key in iterator:
-                op = next(iterator, None)
-                if op is None or op == ',':
-                    kv.append(key)  # A set.
-                else:
-                    assert op == ':'  # A dict.
-                    kv.append((key, next(iterator)))
+                if key == "**":
+                    # dict with pep 448 double-star unpacking
+                    # for now ignoring the values imported by **
+                    next(iterator)
                     next(iterator, None)  # Possible comma.
+                else:
+                    op = next(iterator, None)
+                    if op is None or op == ',':
+                        if key.type == "star_expr":
+                            # pep 448 single-star unpacking
+                            # for now ignoring values imported by *
+                            pass
+                        else:
+                            kv.append(key)  # A set.
+                    else:
+                        assert op == ':'  # A dict.
+                        kv.append((key, next(iterator)))
+                        next(iterator, None)  # Possible comma.
             return kv
         else:
-            return [array_node]
+            if array_node.type == "star_expr":
+                # pep 448 single-star unpacking
+                # for now ignoring values imported by *
+                return []
+            else:
+                return [array_node]
 
     def exact_key_items(self):
         """
@@ -634,12 +653,9 @@ def _check_array_additions(context, sequence):
     return added_types
 
 
-def get_dynamic_array_instance(instance):
+def get_dynamic_array_instance(instance, arguments):
     """Used for set() and list() instances."""
-    if not settings.dynamic_array_additions:
-        return instance.var_args
-
-    ai = _ArrayInstance(instance)
+    ai = _ArrayInstance(instance, arguments)
     from jedi.evaluate import arguments
     return arguments.ValuesArguments([ContextSet(ai)])
 
@@ -655,9 +671,9 @@ class _ArrayInstance(object):
     and therefore doesn't need filters, `py__bool__` and so on, because
     we don't use these operations in `builtins.py`.
     """
-    def __init__(self, instance):
+    def __init__(self, instance, var_args):
         self.instance = instance
-        self.var_args = instance.var_args
+        self.var_args = var_args
 
     def py__iter__(self):
         var_args = self.var_args

jedi/evaluate/context/klass.py

@@ -38,11 +38,12 @@ py__doc__(include_call_signature:      Returns the docstring for a context.
 
 """
 from jedi._compatibility import use_metaclass
+from jedi.parser_utils import get_parent_scope
 from jedi.evaluate.cache import evaluator_method_cache, CachedMetaClass
 from jedi.evaluate import compiled
 from jedi.evaluate.lazy_context import LazyKnownContext
 from jedi.evaluate.filters import ParserTreeFilter, TreeNameDefinition, \
-    ContextName, AnonymousInstanceParamName
+    ContextName
 from jedi.evaluate.base_context import ContextSet, iterator_to_context_set, \
     TreeContext
 
@@ -58,9 +59,10 @@ def apply_py__get__(context, base_context):
 
 
 class ClassName(TreeNameDefinition):
-    def __init__(self, parent_context, tree_name, name_context):
+    def __init__(self, parent_context, tree_name, name_context, apply_decorators):
         super(ClassName, self).__init__(parent_context, tree_name)
         self._name_context = name_context
+        self._apply_decorators = apply_decorators
 
     @iterator_to_context_set
     def infer(self):
@@ -72,16 +74,45 @@ class ClassName(TreeNameDefinition):
             self.parent_context.evaluator, self._name_context, self.tree_name)
 
         for result_context in inferred:
-            for c in apply_py__get__(result_context, self.parent_context):
-                yield c
+            if self._apply_decorators:
+                for c in apply_py__get__(result_context, self.parent_context):
+                    yield c
+            else:
+                yield result_context
 
 
 class ClassFilter(ParserTreeFilter):
     name_class = ClassName
 
+    def __init__(self, *args, **kwargs):
+        self._is_instance = kwargs.pop('is_instance')  # Python 2 :/
+        super(ClassFilter, self).__init__(*args, **kwargs)
+
     def _convert_names(self, names):
-        return [self.name_class(self.context, name, self._node_context)
-                for name in names]
+        return [
+            self.name_class(
+                parent_context=self.context,
+                tree_name=name,
+                name_context=self._node_context,
+                apply_decorators=not self._is_instance,
+            ) for name in names
+        ]
+
+    def _equals_origin_scope(self):
+        node = self._origin_scope
+        while node is not None:
+            if node == self._parser_scope or node == self.context:
+                return True
+            node = get_parent_scope(node)
+        return False
+
+    def _access_possible(self, name):
+        return not name.value.startswith('__') or name.value.endswith('__') \
+            or self._equals_origin_scope()
+
+    def _filter(self, names):
+        names = super(ClassFilter, self)._filter(names)
+        return [name for name in names if self._access_possible(name)]
 
 
 class ClassContext(use_metaclass(CachedMetaClass, TreeContext)):
@@ -91,10 +122,6 @@ class ClassContext(use_metaclass(CachedMetaClass, TreeContext)):
     """
     api_type = u'class'
 
-    def __init__(self, evaluator, parent_context, classdef):
-        super(ClassContext, self).__init__(evaluator, parent_context=parent_context)
-        self.tree_node = classdef
-
     @evaluator_method_cache(default=())
     def py__mro__(self):
         def add(cls):
@@ -148,11 +175,6 @@ class ClassContext(use_metaclass(CachedMetaClass, TreeContext)):
     def py__class__(self):
         return compiled.builtin_from_name(self.evaluator, u'type')
 
-    def get_params(self):
-        from jedi.evaluate.context import AnonymousInstance
-        anon = AnonymousInstance(self.evaluator, self.parent_context, self)
-        return [AnonymousInstanceParamName(anon, param.name) for param in self.funcdef.get_params()]
-
     def get_filters(self, search_global, until_position=None, origin_scope=None, is_instance=False):
         if search_global:
             yield ParserTreeFilter(
@@ -169,7 +191,9 @@ class ClassContext(use_metaclass(CachedMetaClass, TreeContext)):
                 else:
                     yield ClassFilter(
                         self.evaluator, self, node_context=cls,
-                        origin_scope=origin_scope)
+                        origin_scope=origin_scope,
+                        is_instance=is_instance
+                    )
 
     def is_class(self):
         return True

jedi/evaluate/context/module.py

@@ -43,8 +43,11 @@ class ModuleContext(TreeContext):
     parent_context = None
 
     def __init__(self, evaluator, module_node, path, code_lines):
-        super(ModuleContext, self).__init__(evaluator, parent_context=None)
-        self.tree_node = module_node
+        super(ModuleContext, self).__init__(
+            evaluator,
+            parent_context=None,
+            tree_node=module_node
+        )
         self._path = path
         self.code_lines = code_lines
 

jedi/evaluate/context/namespace.py

@@ -4,7 +4,7 @@ from itertools import chain
 from jedi.evaluate.cache import evaluator_method_cache
 from jedi.evaluate import imports
 from jedi.evaluate.filters import DictFilter, AbstractNameDefinition, ContextNameMixin
-from jedi.evaluate.base_context import TreeContext, ContextSet
+from jedi.evaluate.base_context import Context
 
 
 class ImplicitNSName(ContextNameMixin, AbstractNameDefinition):
@@ -17,7 +17,7 @@ class ImplicitNSName(ContextNameMixin, AbstractNameDefinition):
         self.string_name = string_name
 
 
-class ImplicitNamespaceContext(TreeContext):
+class ImplicitNamespaceContext(Context):
     """
     Provides support for implicit namespace packages
     """

jedi/evaluate/docstrings.py

@@ -68,7 +68,7 @@ def _search_param_in_numpydocstr(docstr, param_str):
         return []
     for p_name, p_type, p_descr in params:
         if p_name == param_str:
-            m = re.match('([^,]+(,[^,]+)*?)(,[ ]*optional)?$', p_type)
+            m = re.match(r'([^,]+(,[^,]+)*?)(,[ ]*optional)?$', p_type)
             if m:
                 p_type = m.group(1)
             return list(_expand_typestr(p_type))
@@ -103,11 +103,11 @@ def _expand_typestr(type_str):
     Attempts to interpret the possible types in `type_str`
     """
     # Check if alternative types are specified with 'or'
-    if re.search('\\bor\\b', type_str):
+    if re.search(r'\bor\b', type_str):
         for t in type_str.split('or'):
             yield t.split('of')[0].strip()
     # Check if like "list of `type`" and set type to list
-    elif re.search('\\bof\\b', type_str):
+    elif re.search(r'\bof\b', type_str):
         yield type_str.split('of')[0]
     # Check if type has is a set of valid literal values eg: {'C', 'F', 'A'}
     elif type_str.startswith('{'):
@@ -194,7 +194,7 @@ def _evaluate_for_statement_string(module_context, string):
     if string is None:
         return []
 
-    for element in re.findall('((?:\w+\.)*\w+)\.', string):
+    for element in re.findall(r'((?:\w+\.)*\w+)\.', string):
         # Try to import module part in dotted name.
         # (e.g., 'threading' in 'threading.Thread').
         string = 'import %s\n' % element + string
@@ -266,7 +266,8 @@ def _execute_array_values(evaluator, array):
 
 @evaluator_method_cache()
 def infer_param(execution_context, param):
-    from jedi.evaluate.context.instance import AnonymousInstanceFunctionExecution
+    from jedi.evaluate.context.instance import InstanceArguments
+    from jedi.evaluate.context import FunctionExecutionContext
 
     def eval_docstring(docstring):
         return ContextSet.from_iterable(
@@ -280,9 +281,10 @@ def infer_param(execution_context, param):
         return NO_CONTEXTS
 
     types = eval_docstring(execution_context.py__doc__())
-    if isinstance(execution_context, AnonymousInstanceFunctionExecution) and \
-            execution_context.function_context.name.string_name == '__init__':
-        class_context = execution_context.instance.class_context
+    if isinstance(execution_context, FunctionExecutionContext) \
+            and isinstance(execution_context.var_args, InstanceArguments) \
+            and execution_context.function_context.py__name__() == '__init__':
+        class_context = execution_context.var_args.instance.class_context
         types |= eval_docstring(class_context.py__doc__())
 
     return types

jedi/evaluate/dynamic.py

@@ -28,22 +28,30 @@ from jedi.evaluate.helpers import is_stdlib_path
 from jedi.evaluate.utils import to_list
 from jedi.parser_utils import get_parent_scope
 from jedi.evaluate.context import ModuleContext, instance
-from jedi.evaluate.base_context import ContextSet
-
+from jedi.evaluate.base_context import ContextSet, NO_CONTEXTS
+from jedi.evaluate import recursion
 
 
 MAX_PARAM_SEARCHES = 20
 
 
-class MergedExecutedParams(object):
+class DynamicExecutedParams(object):
     """
     Simulates being a parameter while actually just being multiple params.
     """
-    def __init__(self, executed_params):
+
+    def __init__(self, evaluator, executed_params):
+        self.evaluator = evaluator
         self._executed_params = executed_params
 
     def infer(self):
-        return ContextSet.from_sets(p.infer() for p in self._executed_params)
+        with recursion.execution_allowed(self.evaluator, self) as allowed:
+            # We need to catch recursions that may occur, because an
+            # anonymous functions can create an anonymous parameter that is
+            # more or less self referencing.
+            if allowed:
+                return ContextSet.from_sets(p.infer() for p in self._executed_params)
+            return NO_CONTEXTS
 
 
 @debug.increase_indent
@@ -91,10 +99,10 @@ def search_params(evaluator, execution_context, funcdef):
             )
             if function_executions:
                 zipped_params = zip(*list(
-                    function_execution.get_params()
+                    function_execution.get_executed_params()
                     for function_execution in function_executions
                 ))
-                params = [MergedExecutedParams(executed_params) for executed_params in zipped_params]
+                params = [DynamicExecutedParams(evaluator, executed_params) for executed_params in zipped_params]
                 # Evaluate the ExecutedParams to types.
             else:
                 return create_default_params(execution_context, funcdef)
@@ -200,7 +208,7 @@ def _check_name_for_execution(evaluator, context, compare_node, name, trailer):
             # Here we're trying to find decorators by checking the first
             # parameter. It's not very generic though. Should find a better
             # solution that also applies to nested decorators.
-            params = value.parent_context.get_params()
+            params = value.parent_context.get_executed_params()
             if len(params) != 1:
                 continue
             values = params[0].infer()

jedi/evaluate/filters.py

@@ -38,12 +38,6 @@ class AbstractNameDefinition(object):
             return '<%s: %s>' % (self.__class__.__name__, self.string_name)
         return '<%s: %s@%s>' % (self.__class__.__name__, self.string_name, self.start_pos)
 
-    def execute(self, arguments):
-        return self.infer().execute(arguments)
-
-    def execute_evaluated(self, *args, **kwargs):
-        return self.infer().execute_evaluated(*args, **kwargs)
-
     def is_import(self):
         return False
 
@@ -142,24 +136,11 @@ class ParamName(AbstractTreeName):
         return self.get_param().infer()
 
     def get_param(self):
-        params = self.parent_context.get_params()
+        params = self.parent_context.get_executed_params()
         param_node = search_ancestor(self.tree_name, 'param')
         return params[param_node.position_index]
 
 
-class AnonymousInstanceParamName(ParamName):
-    def infer(self):
-        param_node = search_ancestor(self.tree_name, 'param')
-        # TODO I think this should not belong here. It's not even really true,
-        #      because classmethod and other descriptors can change it.
-        if param_node.position_index == 0:
-            # This is a speed optimization, to return the self param (because
-            # it's known). This only affects anonymous instances.
-            return ContextSet(self.parent_context.instance)
-        else:
-            return self.get_param().infer()
-
-
 class AbstractFilter(object):
     _until_position = None
 
@@ -272,10 +253,6 @@ class FunctionExecutionFilter(ParserTreeFilter):
                 yield TreeNameDefinition(self.context, name)
 
 
-class AnonymousInstanceFunctionExecutionFilter(FunctionExecutionFilter):
-    param_name = AnonymousInstanceParamName
-
-
 class GlobalNameFilter(AbstractUsedNamesFilter):
     def __init__(self, context, parser_scope):
         super(GlobalNameFilter, self).__init__(context, parser_scope)

jedi/evaluate/helpers.py

@@ -19,7 +19,7 @@ def is_stdlib_path(path):
         return False
 
     base_path = os.path.join(sys.prefix, 'lib', 'python')
-    return bool(re.match(re.escape(base_path) + '\d.\d', path))
+    return bool(re.match(re.escape(base_path) + r'\d.\d', path))
 
 
 def deep_ast_copy(obj):
@@ -64,6 +64,10 @@ def evaluate_call_of_leaf(context, leaf, cut_own_trailer=False):
     The option ``cut_own_trailer`` must be set to true for the second purpose.
     """
     trailer = leaf.parent
+    if trailer.type == 'fstring':
+        from jedi.evaluate import compiled
+        return compiled.get_string_context_set(context.evaluator)
+
     # The leaf may not be the last or first child, because there exist three
     # different trailers: `( x )`, `[ x ]` and `.x`. In the first two examples
     # we should not match anything more than x.

jedi/evaluate/imports.py

@@ -26,7 +26,7 @@ from jedi.evaluate import sys_path
 from jedi.evaluate import helpers
 from jedi.evaluate import compiled
 from jedi.evaluate import analysis
-from jedi.evaluate.utils import unite, dotted_from_fs_path
+from jedi.evaluate.utils import unite
 from jedi.evaluate.cache import evaluator_method_cache
 from jedi.evaluate.filters import AbstractNameDefinition
 from jedi.evaluate.base_context import ContextSet, NO_CONTEXTS
@@ -264,8 +264,11 @@ class Importer(object):
         )
 
     def sys_path_with_modifications(self):
-        sys_path_mod = self._evaluator.get_sys_path() \
-                       + sys_path.check_sys_path_modifications(self.module_context)
+
+        sys_path_mod = (
+            self._evaluator.get_sys_path()
+            + sys_path.check_sys_path_modifications(self.module_context)
+        )
 
         if self.import_path and self.file_path is not None \
                 and self._evaluator.environment.version_info.major == 2:
@@ -275,8 +278,9 @@ class Importer(object):
         return sys_path_mod
 
     def follow(self):
-        if not self.import_path:
+        if not self.import_path or not self._evaluator.infer_enabled:
             return NO_CONTEXTS
+
         return self._do_import(self.import_path, self.sys_path_with_modifications())
 
     def _do_import(self, import_path, sys_path):
@@ -300,6 +304,14 @@ class Importer(object):
                 # Old style
                 return self._do_import(('flaskext',) + import_path[2:], sys_path)
 
+        if import_parts[0] in settings.auto_import_modules:
+            module = _load_module(
+                self._evaluator,
+                import_names=import_parts,
+                sys_path=sys_path,
+            )
+            return ContextSet(module)
+
         module_name = '.'.join(import_parts)
         try:
             return ContextSet(self._evaluator.module_cache.get(module_name))
@@ -341,7 +353,8 @@ class Importer(object):
                     code, module_path, is_pkg = self._evaluator.compiled_subprocess.get_module_info(
                         string=import_parts[-1],
                         path=path,
-                        full_name=module_name
+                        full_name=module_name,
+                        is_global_search=False,
                     )
                     if module_path is not None:
                         break
@@ -349,13 +362,14 @@ class Importer(object):
                     _add_error(self.module_context, import_path[-1])
                     return NO_CONTEXTS
         else:
-            debug.dbg('search_module %s in %s', import_parts[-1], self.file_path)
+            debug.dbg('global search_module %s in %s', import_parts[-1], self.file_path)
             # Override the sys.path. It works only good that way.
             # Injecting the path directly into `find_module` did not work.
             code, module_path, is_pkg = self._evaluator.compiled_subprocess.get_module_info(
                 string=import_parts[-1],
                 full_name=module_name,
                 sys_path=sys_path,
+                is_global_search=True,
             )
             if module_path is None:
                 # The module is not a package.
@@ -364,7 +378,7 @@ class Importer(object):
 
         module = _load_module(
             self._evaluator, module_path, code, sys_path,
-            module_name=module_name,
+            import_names=import_parts,
             safe_module_name=True,
         )
 
@@ -464,9 +478,13 @@ class Importer(object):
 
 
 def _load_module(evaluator, path=None, code=None, sys_path=None,
-                 module_name=None, safe_module_name=False):
+                 import_names=None, safe_module_name=False):
+    if import_names is None:
+        dotted_name = None
+    else:
+        dotted_name = '.'.join(import_names)
     try:
-        return evaluator.module_cache.get(module_name)
+        return evaluator.module_cache.get(dotted_name)
     except KeyError:
         pass
     try:
@@ -485,12 +503,10 @@ def _load_module(evaluator, path=None, code=None, sys_path=None,
         if sys_path is None:
             sys_path = evaluator.get_sys_path()
 
-        dotted_path = path and dotted_from_fs_path(path, sys_path)
-        if path is not None and path.endswith(('.py', '.zip', '.egg')) \
-                and dotted_path not in settings.auto_import_modules:
-
+        if path is not None and path.endswith(('.py', '.zip', '.egg')):
             module_node = evaluator.parse(
-                code=code, path=path, cache=True, diff_cache=True,
+                code=code, path=path, cache=True,
+                diff_cache=settings.fast_parser,
                 cache_path=settings.cache_directory)
 
             from jedi.evaluate.context import ModuleContext
@@ -500,10 +516,11 @@ def _load_module(evaluator, path=None, code=None, sys_path=None,
                 code_lines=get_cached_code_lines(evaluator.grammar, path),
             )
         else:
-            module = compiled.load_module(evaluator, path=path, sys_path=sys_path)
+            assert dotted_name is not None
+            module = compiled.load_module(evaluator, dotted_name=dotted_name, sys_path=sys_path)
 
-    if module is not None and module_name is not None:
-        add_module_to_cache(evaluator, module_name, module, safe=safe_module_name)
+    if module is not None and dotted_name is not None:
+        add_module_to_cache(evaluator, dotted_name, module, safe=safe_module_name)
 
     return module
 
@@ -542,10 +559,11 @@ def get_modules_containing_name(evaluator, modules, name):
             code = python_bytes_to_unicode(f.read(), errors='replace')
             if name in code:
                 e_sys_path = evaluator.get_sys_path()
-                module_name = sys_path.dotted_path_in_sys_path(e_sys_path, path)
+                import_names = sys_path.dotted_path_in_sys_path(e_sys_path, path)
                 module = _load_module(
                     evaluator, path, code,
-                    sys_path=e_sys_path, module_name=module_name
+                    sys_path=e_sys_path,
+                    import_names=import_names,
                 )
                 return module
 

jedi/evaluate/lazy_context.py

@@ -1,6 +1,7 @@
 from jedi.evaluate.base_context import ContextSet, NO_CONTEXTS
 from jedi.common.utils import monkeypatch
 
+
 class AbstractLazyContext(object):
     def __init__(self, data):
         self.data = data

jedi/evaluate/param.py

@@ -41,7 +41,7 @@ class ExecutedParam(object):
         return '<%s: %s>' % (self.__class__.__name__, self.string_name)
 
 
-def get_params(execution_context, var_args):
+def get_executed_params(execution_context, var_args):
     result_params = []
     param_dict = {}
     funcdef = execution_context.tree_node

jedi/evaluate/pep0484.py

@@ -157,8 +157,8 @@ def infer_param(execution_context, param):
                 "Comments length != Params length %s %s",
                 params_comments, all_params
             )
-        from jedi.evaluate.context.instance import BaseInstanceFunctionExecution
-        if isinstance(execution_context, BaseInstanceFunctionExecution):
+        from jedi.evaluate.context.instance import InstanceArguments
+        if isinstance(execution_context.var_args, InstanceArguments):
             if index == 0:
                 # Assume it's self, which is already handled
                 return NO_CONTEXTS

jedi/evaluate/recursion.py

@@ -65,7 +65,7 @@ def execution_allowed(evaluator, node):
 
     if node in pushed_nodes:
         debug.warning('catched stmt recursion: %s @%s', node,
-                      node.start_pos)
+                      getattr(node, 'start_pos', None))
         yield False
     else:
         try:
@@ -77,14 +77,14 @@ def execution_allowed(evaluator, node):
 
 def execution_recursion_decorator(default=NO_CONTEXTS):
     def decorator(func):
-        def wrapper(execution, **kwargs):
-            detector = execution.evaluator.execution_recursion_detector
-            allowed = detector.push_execution(execution)
+        def wrapper(self, **kwargs):
+            detector = self.evaluator.execution_recursion_detector
+            allowed = detector.push_execution(self)
             try:
                 if allowed:
                     result = default
                 else:
-                    result = func(execution, **kwargs)
+                    result = func(self, **kwargs)
             finally:
                 detector.pop_execution()
             return result

jedi/evaluate/stdlib.py

@@ -9,21 +9,18 @@ Note that this module exists only to implement very specific functionality in
 the standard library. The usual way to understand the standard library is the
 compiled module that returns the types for C-builtins.
 """
-import re
-
 import parso
 
 from jedi._compatibility import force_unicode
 from jedi import debug
-from jedi.evaluate.arguments import ValuesArguments
+from jedi.evaluate.arguments import ValuesArguments, repack_with_argument_clinic
 from jedi.evaluate import analysis
 from jedi.evaluate import compiled
-from jedi.evaluate.context.instance import InstanceFunctionExecution, \
-    AbstractInstanceContext, CompiledInstance, BoundMethod, \
-    AnonymousInstanceFunctionExecution
+from jedi.evaluate.context.instance import \
+    AbstractInstanceContext, CompiledInstance, BoundMethod, InstanceArguments
 from jedi.evaluate.base_context import ContextualizedNode, \
     NO_CONTEXTS, ContextSet
-from jedi.evaluate.context import ClassContext, ModuleContext
+from jedi.evaluate.context import ClassContext, ModuleContext, FunctionExecutionContext
 from jedi.evaluate.context import iterable
 from jedi.evaluate.lazy_context import LazyTreeContext
 from jedi.evaluate.syntax_tree import is_string
@@ -72,7 +69,7 @@ def execute(evaluator, obj, arguments):
         except KeyError:
             pass
         else:
-            return func(evaluator, obj, arguments)
+            return func(evaluator, obj, arguments=arguments)
     raise NotInStdLib()
 
 
@@ -89,42 +86,22 @@ def argument_clinic(string, want_obj=False, want_context=False, want_arguments=F
     """
     Works like Argument Clinic (PEP 436), to validate function params.
     """
-    clinic_args = []
-    allow_kwargs = False
-    optional = False
-    while string:
-        # Optional arguments have to begin with a bracket. And should always be
-        # at the end of the arguments. This is therefore not a proper argument
-        # clinic implementation. `range()` for exmple allows an optional start
-        # value at the beginning.
-        match = re.match('(?:(?:(\[),? ?|, ?|)(\w+)|, ?/)\]*', string)
-        string = string[len(match.group(0)):]
-        if not match.group(2):  # A slash -> allow named arguments
-            allow_kwargs = True
-            continue
-        optional = optional or bool(match.group(1))
-        word = match.group(2)
-        clinic_args.append((word, optional, allow_kwargs))
 
     def f(func):
-        def wrapper(evaluator, obj, arguments):
+        @repack_with_argument_clinic(string, keep_arguments_param=True)
+        def wrapper(evaluator, obj, *args, **kwargs):
+            arguments = kwargs.pop('arguments')
+            assert not kwargs  # Python 2...
             debug.dbg('builtin start %s' % obj, color='MAGENTA')
             result = NO_CONTEXTS
-            try:
-                lst = list(arguments.eval_argument_clinic(clinic_args))
-            except ValueError:
-                pass
-            else:
-                kwargs = {}
-                if want_context:
-                    kwargs['context'] = arguments.context
-                if want_obj:
-                    kwargs['obj'] = obj
-                if want_arguments:
-                    kwargs['arguments'] = arguments
-                result = func(evaluator, *lst, **kwargs)
-            finally:
-                debug.dbg('builtin end: %s', result, color='MAGENTA')
+            if want_context:
+                kwargs['context'] = arguments.context
+            if want_obj:
+                kwargs['obj'] = obj
+            if want_arguments:
+                kwargs['arguments'] = arguments
+            result = func(evaluator, *args, **kwargs)
+            debug.dbg('builtin end: %s', result, color='MAGENTA')
             return result
 
         return wrapper
@@ -187,10 +164,11 @@ class SuperInstance(AbstractInstanceContext):
 @argument_clinic('[type[, obj]], /', want_context=True)
 def builtins_super(evaluator, types, objects, context):
     # TODO make this able to detect multiple inheritance super
-    if isinstance(context, (InstanceFunctionExecution,
-                            AnonymousInstanceFunctionExecution)):
-        su = context.instance.py__class__().py__bases__()
-        return su[0].infer().execute_evaluated()
+    if isinstance(context, FunctionExecutionContext):
+        if isinstance(context.var_args, InstanceArguments):
+            su = context.var_args.instance.py__class__().py__bases__()
+            return su[0].infer().execute_evaluated()
+
     return NO_CONTEXTS
 
 
@@ -334,8 +312,8 @@ _implemented = {
         'deepcopy': _return_first_param,
     },
     'json': {
-        'load': lambda *args: NO_CONTEXTS,
-        'loads': lambda *args: NO_CONTEXTS,
+        'load': lambda evaluator, obj, arguments: NO_CONTEXTS,
+        'loads': lambda evaluator, obj, arguments: NO_CONTEXTS,
     },
     'collections': {
         'namedtuple': collections_namedtuple,

jedi/evaluate/syntax_tree.py

@@ -71,7 +71,7 @@ def eval_node(context, element):
     if typ in ('name', 'number', 'string', 'atom', 'strings', 'keyword'):
         return eval_atom(context, element)
     elif typ == 'lambdef':
-        return ContextSet(FunctionContext(evaluator, context, element))
+        return ContextSet(FunctionContext.from_context(context, element))
     elif typ == 'expr_stmt':
         return eval_expr_stmt(context, element)
     elif typ in ('power', 'atom_expr'):
@@ -255,7 +255,8 @@ def eval_atom(context, atom):
             array_node_c = array_node.children
         except AttributeError:
             array_node_c = []
-        if c[0] == '{' and (array_node == '}' or ':' in array_node_c):
+        if c[0] == '{' and (array_node == '}' or ':' in array_node_c or
+                            '**' in array_node_c):
             context = iterable.DictLiteralContext(context.evaluator, context, atom)
         else:
             context = iterable.SequenceLiteralContext(context.evaluator, context, atom)
@@ -270,7 +271,7 @@ def eval_expr_stmt(context, stmt, seek_name=None):
         # necessary.
         if not allowed and context.get_root_context() == context.evaluator.builtins_module:
             try:
-                instance = context.instance
+                instance = context.var_args.instance
             except AttributeError:
                 pass
             else:
@@ -580,14 +581,10 @@ def _apply_decorators(context, node):
         decoratee_context = ClassContext(
             context.evaluator,
             parent_context=context,
-            classdef=node
+            tree_node=node
         )
     else:
-        decoratee_context = FunctionContext(
-            context.evaluator,
-            parent_context=context,
-            funcdef=node
-        )
+        decoratee_context = FunctionContext.from_context(context, node)
     initial = values = ContextSet(decoratee_context)
     for dec in reversed(node.get_decorators()):
         debug.dbg('decorator: %s %s', dec, values)

jedi/evaluate/sys_path.py

@@ -198,7 +198,7 @@ def _get_buildout_script_paths(search_path):
 
 def dotted_path_in_sys_path(sys_path, module_path):
     """
-    Returns the dotted path inside a sys.path.
+    Returns the dotted path inside a sys.path as a list of names.
     """
     # First remove the suffix.
     for suffix in all_suffixes():
@@ -221,6 +221,6 @@ def dotted_path_in_sys_path(sys_path, module_path):
                 for string in split:
                     if not string or '.' in string:
                         return None
-                return '.'.join(split)
+                return split
 
     return None

jedi/evaluate/utils.py

@@ -11,7 +11,7 @@ from jedi._compatibility import reraise
 _sep = os.path.sep
 if os.path.altsep is not None:
     _sep += os.path.altsep
-_path_re = re.compile('(?:\.[^{0}]+|[{0}]__init__\.py)$'.format(re.escape(_sep)))
+_path_re = re.compile(r'(?:\.[^{0}]+|[{0}]__init__\.py)$'.format(re.escape(_sep)))
 del _sep
 
 
@@ -117,38 +117,3 @@ def indent_block(text, indention='    '):
         text = text[:-1]
     lines = text.split('\n')
     return '\n'.join(map(lambda s: indention + s, lines)) + temp
-
-
-def dotted_from_fs_path(fs_path, sys_path):
-    """
-    Changes `/usr/lib/python3.4/email/utils.py` to `email.utils`.  I.e.
-    compares the path with sys.path and then returns the dotted_path. If the
-    path is not in the sys.path, just returns None.
-    """
-    if os.path.basename(fs_path).startswith('__init__.'):
-        # We are calculating the path. __init__ files are not interesting.
-        fs_path = os.path.dirname(fs_path)
-
-    # prefer
-    #   - UNIX
-    #     /path/to/pythonX.Y/lib-dynload
-    #     /path/to/pythonX.Y/site-packages
-    #   - Windows
-    #     C:\path\to\DLLs
-    #     C:\path\to\Lib\site-packages
-    # over
-    #   - UNIX
-    #     /path/to/pythonX.Y
-    #   - Windows
-    #     C:\path\to\Lib
-    path = ''
-    for s in sys_path:
-        if (fs_path.startswith(s) and len(path) < len(s)):
-            path = s
-
-    # - Window
-    # X:\path\to\lib-dynload/datetime.pyd => datetime
-    module_path = fs_path[len(path):].lstrip(os.path.sep).lstrip('/')
-    # - Window
-    # Replace like X:\path\to\something/foo/bar.py
-    return _path_re.sub('', module_path).replace(os.path.sep, '.').replace('/', '.')

jedi/parser_utils.py

@@ -1,3 +1,4 @@
+import re
 import textwrap
 from inspect import cleandoc
 
@@ -158,6 +159,7 @@ def get_call_signature(funcdef, width=72, call_string=None):
         p = '(' + ''.join(param.get_code() for param in funcdef.get_params()).strip() + ')'
     else:
         p = funcdef.children[2].get_code()
+    p = re.sub(r'\s+', ' ', p)
     if funcdef.annotation:
         rtype = " ->" + funcdef.annotation.get_code()
     else:
@@ -183,6 +185,8 @@ def get_doc_with_call_signature(scope_node):
     doc = clean_scope_docstring(scope_node)
     if call_signature is None:
         return doc
+    if not doc:
+        return call_signature
     return '%s\n\n%s' % (call_signature, doc)
 
 

jedi/settings.py

@@ -144,7 +144,8 @@ Check for `isinstance` and other information to infer a type.
 """
 
 auto_import_modules = [
-    'hashlib',  # setattr
+    'hashlib',  # hashlib is mostly using setattr, which jedi doesn't understand
+    'gi',  # This third-party repository (GTK stuff) doesn't really work with jedi
 ]
 """
 Modules that are not analyzed but imported, although they contain Python code.

jedi/utils.py

@@ -113,7 +113,7 @@ def setup_readline(namespace_module=__main__):
         # this code. This didn't use to be an issue until 3.3. Starting with
         # 3.4 this is different, it always overwrites the completer if it's not
         # already imported here.
-        import rlcompleter
+        import rlcompleter  # noqa: F401
         import readline
     except ImportError:
         print("Jedi: Module readline not available.")
@@ -138,5 +138,5 @@ def version_info():
     """
     Version = namedtuple('Version', 'major, minor, micro')
     from jedi import __version__
-    tupl = re.findall('[a-z]+|\d+', __version__)
+    tupl = re.findall(r'[a-z]+|\d+', __version__)
     return Version(*[x if i == 3 else int(x) for i, x in enumerate(tupl)])

setup.cfg

@@ -1,2 +1,8 @@
 [bdist_wheel]
 universal=1
+
+[flake8]
+max-line-length = 100
+ignore =
+  # do not use bare 'except'
+  E722,

setup.py

@@ -29,9 +29,17 @@ setup(name='jedi',
       keywords='python completion refactoring vim',
       long_description=readme,
       packages=find_packages(exclude=['test', 'test.*']),
-      python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*',
+      python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*',
       install_requires=install_requires,
-      extras_require={'dev': ['docopt']},
+      extras_require={
+          'testing': [
+              'pytest>=3.1.0',
+              # docopt for sith doctests
+              'docopt',
+              # coloroma for colored debug output
+              'colorama',
+          ],
+      },
       package_data={'jedi': ['evaluate/compiled/fake/*.pym']},
       platforms=['any'],
       classifiers=[
@@ -43,7 +51,6 @@ setup(name='jedi',
           'Programming Language :: Python :: 2',
           'Programming Language :: Python :: 2.7',
           'Programming Language :: Python :: 3',
-          'Programming Language :: Python :: 3.3',
           'Programming Language :: Python :: 3.4',
           'Programming Language :: Python :: 3.5',
           'Programming Language :: Python :: 3.6',

test/completion/arrays.py

@@ -437,7 +437,7 @@ def test_func():
 #? int()
 tuple({1})[0]
 
-# python >= 3.3
+# python >= 3.4
 # -----------------
 # PEP 3132 Extended Iterable Unpacking (star unpacking)
 # -----------------

test/completion/basic.py

@@ -154,6 +154,9 @@ def global_define():
 #? int()
 global_var_in_func
 
+#? ['global_var_in_func']
+global_var_in_f
+
 
 def funct1():
     # From issue #610
@@ -175,6 +178,7 @@ def init_global_var_predefined():
 #? int() None
 global_var_predefined
 
+
 # -----------------
 # within docstrs
 # -----------------

test/completion/classes.py

@@ -417,6 +417,9 @@ class PrivateVar():
     def __private_func(self):
         return 1
 
+    #? int()
+    __private_func()
+
     def wrap_private(self):
         return self.__private_func()
 #? []
@@ -425,6 +428,8 @@ PrivateVar().__var
 PrivateVar().__var
 #? []
 PrivateVar().__private_func
+#? []
+PrivateVar.__private_func
 #? int()
 PrivateVar().wrap_private()
 

test/completion/comprehensions.py

@@ -52,12 +52,12 @@ left
 [a for a in {1:'x'}][0]
 
 # list comprehensions should also work in combination with functions
-def listen(arg):
+def _listen(arg):
     for x in arg:
         #? str()
         x
 
-listen(['' for x in [1]])
+_listen(['' for x in [1]])
 #?
 ([str for x in []])[0]
 
@@ -212,3 +212,14 @@ next(iter({a for a in range(10)}))
 
 #? int()
 [a for a in {1, 2, 3}][0]
+
+# -----------------
+# syntax errors
+# -----------------
+
+# Issue #1146
+
+#? ['list']
+[int(str(x.value) for x in list
+
+def reset_missing_bracket(): pass

test/completion/fstring.py

@@ -25,3 +25,10 @@ Fr'sasdf'
 
 #? 7 str()
 Fr'''sasdf''' + ''
+
+#? ['upper']
+f'xyz'.uppe
+
+
+#? 3 []
+f'f'

test/completion/functions.py

@@ -319,6 +319,7 @@ exe['c']
 a = 'a'
 exe2 = kwargs_func(**{a:3,
                       'b':4.0})
+
 #? int()
 exe2['a']
 #? float()
@@ -326,6 +327,19 @@ exe2['b']
 #? int() float()
 exe2['c']
 
+exe3 = kwargs_func(**{k: v for k, v in [(a, 3), ('b', 4.0)]})
+
+# Should resolve to the same as 2 but jedi is not smart enough yet
+# Here to make sure it doesn't result in crash though
+#? 
+exe3['a']
+
+#? 
+exe3['b']
+
+#? 
+exe3['c']
+
 # -----------------
 # *args / ** kwargs
 # -----------------

test/completion/generators.py

@@ -220,7 +220,7 @@ def x():
 # yield from
 # -----------------
 
-# python >= 3.3
+# python >= 3.4
 
 def yield_from():
     yield from iter([1])

test/completion/pep0484_basic.py

@@ -1,6 +1,6 @@
 """ Pep-0484 type hinting """
 
-# python >= 3.2
+# python >= 3.4
 
 
 class A():

test/completion/pep0484_typing.py

@@ -243,7 +243,7 @@ for key in x.keys():
 for value in x.values():
     #? int()
     value
-# python >= 3.2
+# python >= 3.4
 
 class TestDefaultDict(typing.DefaultDict[str, int]):
     def setdud(self):
@@ -271,7 +271,7 @@ for key in x.keys():
 for value in x.values():
     #? int()
     value
-# python >= 3.2
+# python >= 3.4
 
 
 """

test/completion/recursion.py

@@ -76,3 +76,18 @@ class InstanceAttributeIfs:
 InstanceAttributeIfs().a1
 #? int() str()
 InstanceAttributeIfs().a2
+
+
+
+class A:
+    def a(self, b):
+        for x in [self.a(i) for i in b]:
+            #?
+            x
+
+class B:
+    def a(self, b):
+        for i in b:
+            for i in self.a(i):
+                #?
+                yield i

test/completion/types.py

@@ -131,3 +131,58 @@ set_t2 = set()
 
 #? ['clear', 'copy']
 set_t2.c
+
+# -----------------
+# pep 448 unpacking generalizations
+# -----------------
+# python >= 3.5
+
+d = {'a': 3}
+dc = {v: 3 for v in ['a']}
+
+#? dict()
+{**d}
+
+#? dict()
+{**dc}
+
+#? str()
+{**d, "b": "b"}["b"]
+
+#? str()
+{**dc, "b": "b"}["b"]
+
+# Should resolve to int() but jedi is not smart enough yet
+# Here to make sure it doesn't result in crash though
+#? 
+{**d}["a"]
+
+# Should resolve to int() but jedi is not smart enough yet
+# Here to make sure it doesn't result in crash though
+#? 
+{**dc}["a"]
+
+s = {1, 2, 3}
+
+#? set()
+{*s}
+
+#? set()
+{*s, 4, *s}
+
+s = {1, 2, 3}
+# Should resolve to int() but jedi is not smart enough yet
+# Here to make sure it doesn't result in crash though
+#? 
+{*s}.pop()
+
+#? int()
+{*s, 4}.pop()
+
+# Should resolve to int() but jedi is not smart enough yet
+# Here to make sure it doesn't result in crash though
+#? 
+[*s][0]
+
+#? int()
+[*s, 4][0]

test/conftest.py

@@ -126,11 +126,12 @@ class StaticAnalysisCase(object):
         return "<%s: %s>" % (self.__class__.__name__, os.path.basename(self._path))
 
 
-@pytest.fixture()
-def venv_path(tmpdir, environment):
+@pytest.fixture(scope='session')
+def venv_path(tmpdir_factory, environment):
     if environment.version_info.major < 3:
         pytest.skip("python -m venv does not exist in Python 2")
 
+    tmpdir = tmpdir_factory.mktemp('venv_path')
     dirname = os.path.join(tmpdir.dirname, 'venv')
 
     # We cannot use the Python from tox because tox creates virtualenvs and

test/examples/namespace_package_relative_import/rel1.py

@@ -0,0 +1 @@
+from .rel2 import name

test/examples/namespace_package_relative_import/rel2.py

@@ -0,0 +1 @@
+name = 1

test/test_api/test_api.py

@@ -285,3 +285,11 @@ def test_backslash_continuation_and_bracket(Script):
     column = lines[-1].index('(')
     def_, = Script(code, line=len(lines), column=column).goto_definitions()
     assert def_.name == 'int'
+
+
+def test_goto_follow_builtin_imports(Script):
+    s = Script('import sys; sys')
+    d, = s.goto_assignments(follow_imports=True)
+    assert d.in_builtin_module() is True
+    d, = s.goto_assignments(follow_imports=True, follow_builtin_imports=True)
+    assert d.in_builtin_module() is False

test/test_api/test_call_signatures.py

@@ -6,7 +6,7 @@ import pytest
 
 from ..helpers import TestCase
 from jedi import cache
-from jedi._compatibility import is_py33
+from jedi._compatibility import is_py3
 
 
 def assert_signature(Script, source, expected_name, expected_index=0, line=None, column=None):
@@ -247,7 +247,7 @@ def _params(Script, source, line=None, column=None):
 
 
 def test_param_name(Script):
-    if not is_py33:
+    if not is_py3:
         p = _params(Script, '''int(''')
         # int is defined as: `int(x[, base])`
         assert p[0].name == 'x'

test/test_api/test_completion.py

@@ -132,7 +132,6 @@ def test_async(Script, environment):
             hey = 3
               ho'''
     )
-    print(code)
     comps = Script(code, column=4).completions()
     names = [c.name for c in comps]
     assert 'foo' in names

test/test_api/test_environment.py

@@ -1,6 +1,5 @@
 import os
 import sys
-from contextlib import contextmanager
 
 import pytest
 
@@ -8,7 +7,7 @@ import jedi
 from jedi._compatibility import py_version
 from jedi.api.environment import get_default_environment, find_virtualenvs, \
     InvalidPythonEnvironment, find_system_environments, \
-    get_system_environment, create_environment
+    get_system_environment, create_environment, get_cached_default_environment
 
 
 def test_sys_path():
@@ -44,7 +43,7 @@ def test_versions(version):
 
 def test_load_module(evaluator):
     access_path = evaluator.compiled_subprocess.load_module(
-        name=u'math',
+        dotted_name=u'math',
         sys_path=evaluator.get_sys_path()
     )
     name, access_handle = access_path.accesses[0]
@@ -85,34 +84,22 @@ def test_killed_subprocess(evaluator, Script):
     assert def_.name == 'str'
 
 
-@contextmanager
-def set_environment_variable(name, value):
-    tmp = os.environ.get(name)
-    try:
-        os.environ[name] = value
-        yield
-    finally:
-        if tmp is None:
-            del os.environ[name]
-        else:
-            os.environ[name] = tmp
-
-
-def test_not_existing_virtualenv():
+def test_not_existing_virtualenv(monkeypatch):
     """Should not match the path that was given"""
     path = '/foo/bar/jedi_baz'
-    with set_environment_variable('VIRTUAL_ENV', path):
-        assert get_default_environment().executable != path
+    monkeypatch.setenv('VIRTUAL_ENV', path)
+    assert get_default_environment().executable != path
 
 
-def test_working_venv(venv_path):
-    with set_environment_variable('VIRTUAL_ENV', venv_path):
-        assert get_default_environment().path == venv_path
+def test_working_venv(venv_path, monkeypatch):
+    monkeypatch.setenv('VIRTUAL_ENV', venv_path)
+    assert get_default_environment().path == venv_path
 
 
 def test_scanning_venvs(venv_path):
     parent_dir = os.path.dirname(venv_path)
-    assert any(venv.path == venv_path for venv in find_virtualenvs([parent_dir]))
+    assert any(venv.path == venv_path
+               for venv in find_virtualenvs([parent_dir]))
 
 
 def test_create_environment_venv_path(venv_path):
@@ -123,3 +110,29 @@ def test_create_environment_venv_path(venv_path):
 def test_create_environment_executable():
     environment = create_environment(sys.executable)
     assert environment.executable == sys.executable
+
+
+def test_get_default_environment_from_env_does_not_use_safe(tmpdir, monkeypatch):
+    fake_python = os.path.join(str(tmpdir), 'fake_python')
+    with open(fake_python, 'w') as f:
+        f.write('')
+
+    def _get_subprocess(self):
+        if self._start_executable != fake_python:
+            raise RuntimeError('Should not get called!')
+        self.executable = fake_python
+        self.path = 'fake'
+
+    monkeypatch.setattr('jedi.api.environment.Environment._get_subprocess',
+                        _get_subprocess)
+
+    monkeypatch.setenv('VIRTUAL_ENV', fake_python)
+    env = get_default_environment()
+    assert env.path == 'fake'
+
+
+def test_changing_venv(venv_path, monkeypatch):
+    monkeypatch.setitem(os.environ, 'VIRTUAL_ENV', venv_path)
+    get_cached_default_environment()
+    monkeypatch.setitem(os.environ, 'VIRTUAL_ENV', sys.executable)
+    assert get_cached_default_environment().executable == sys.executable

test/test_api/test_unicode.py

@@ -72,3 +72,9 @@ def test_wrong_encoding(Script, tmpdir):
 
     c, = Script('import x; x.foo', sys_path=[tmpdir.strpath]).completions()
     assert c.name == 'foobar'
+
+
+def test_encoding_parameter(Script):
+    name = u('hö')
+    s = Script(name.encode('latin-1'), encoding='latin-1')
+    assert s._module_node.get_code() == name

test/test_api/test_usages.py

@@ -1,3 +1,14 @@
 def test_import_usage(Script):
     s = Script("from .. import foo", line=1, column=18, path="foo.py")
     assert [usage.line for usage in s.usages()] == [1]
+
+
+def test_exclude_builtin_modules(Script):
+    def get(include):
+        return [(d.line, d.column) for d in Script(source, column=8).usages(include_builtins=include)]
+    source = '''import sys\nprint(sys.path)'''
+    places = get(include=True)
+    assert places == [(None, None), (1, 7), (2, 6)]
+
+    places = get(include=False)
+    assert places == [(1, 7), (2, 6)]

test/test_evaluate/test_compiled.py

@@ -70,11 +70,7 @@ def test_method_completion(Script, environment):
 
     foo = Foo()
     foo.bar.__func__''')
-    if environment.version_info.major > 2:
-        result = []
-    else:
-        result = ['__func__']
-    assert [c.name for c in Script(code).completions()] == result
+    assert [c.name for c in Script(code).completions()] == ['__func__']
 
 
 def test_time_docstring(Script):

test/test_evaluate/test_docstring.py

@@ -142,6 +142,16 @@ def test_docstring_keyword(Script):
     assert 'assert' in completions[0].docstring()
 
 
+def test_docstring_params_formatting(Script):
+    defs = Script("""
+    def func(param1,
+             param2,
+             param3):
+        pass
+    func""").goto_definitions()
+    assert defs[0].docstring() == 'func(param1, param2, param3)'
+
+
 # ---- Numpy Style Tests ---
 
 @pytest.mark.skipif(numpydoc_unavailable,

test/test_evaluate/test_imports.py

@@ -87,10 +87,10 @@ def test_import_not_in_sys_path(Script):
     ("from flask.ext.", "bar"),
     ("from flask.ext.", "baz"),
     ("from flask.ext.", "moo"),
-    pytest.mark.xfail(("import flask.ext.foo; flask.ext.foo.", "Foo")),
-    pytest.mark.xfail(("import flask.ext.bar; flask.ext.bar.", "Foo")),
-    pytest.mark.xfail(("import flask.ext.baz; flask.ext.baz.", "Foo")),
-    pytest.mark.xfail(("import flask.ext.moo; flask.ext.moo.", "Foo")),
+    pytest.param("import flask.ext.foo; flask.ext.foo.", "Foo", marks=pytest.mark.xfail),
+    pytest.param("import flask.ext.bar; flask.ext.bar.", "Foo", marks=pytest.mark.xfail),
+    pytest.param("import flask.ext.baz; flask.ext.baz.", "Foo", marks=pytest.mark.xfail),
+    pytest.param("import flask.ext.moo; flask.ext.moo.", "Foo", marks=pytest.mark.xfail),
 ])
 def test_flask_ext(Script, code, name):
     """flask.ext.foo is really imported from flaskext.foo or flask_foo.

test/test_evaluate/test_namespace_package.py

@@ -1,6 +1,9 @@
 from os.path import dirname, join
 
 import pytest
+import py
+
+from ..helpers import get_example_dir
 
 
 SYS_PATH = [join(dirname(__file__), d)
@@ -72,3 +75,22 @@ def test_nested_namespace_package(Script):
     result = script.goto_definitions()
 
     assert len(result) == 1
+
+
+def test_relative_import(Script, environment, tmpdir):
+    """
+    Attempt a relative import in a very simple namespace package.
+    """
+    if environment.version_info < (3, 4):
+        pytest.skip()
+
+    directory = get_example_dir('namespace_package_relative_import')
+    # Need to copy the content in a directory where there's no __init__.py.
+    py.path.local(directory).copy(tmpdir)
+    file_path = join(tmpdir.strpath, "rel1.py")
+    script = Script(path=file_path, line=1)
+    d, = script.goto_definitions()
+    assert d.name == 'int'
+    d, = script.goto_assignments()
+    assert d.name == 'name'
+    assert d.module_name == 'rel2'

test/test_evaluate/test_pyc.py

@@ -10,8 +10,10 @@ import os
 import shutil
 import sys
 
+import pytest
+
 import jedi
-from ..helpers import cwd_at
+from jedi.api.environment import SameEnvironment
 
 
 SRC = """class Foo:
@@ -22,35 +24,44 @@ class Bar:
 """
 
 
-def generate_pyc():
-    os.mkdir("dummy_package")
-    with open("dummy_package/__init__.py", 'w'):
+@pytest.fixture
+def pyc_project_path(tmpdir):
+    path = tmpdir.strpath
+    dummy_package_path = os.path.join(path, "dummy_package")
+    os.mkdir(dummy_package_path)
+    with open(os.path.join(dummy_package_path, "__init__.py"), 'w'):
         pass
-    with open("dummy_package/dummy.py", 'w') as f:
+
+    dummy_path = os.path.join(dummy_package_path, 'dummy.py')
+    with open(dummy_path, 'w') as f:
         f.write(SRC)
     import compileall
-    compileall.compile_file("dummy_package/dummy.py")
-    os.remove("dummy_package/dummy.py")
+    compileall.compile_file(dummy_path)
+    os.remove(dummy_path)
 
-    if sys.version_info[0] == 3:
+    if sys.version_info.major == 3:
         # Python3 specific:
         # To import pyc modules, we must move them out of the __pycache__
         # directory and rename them to remove ".cpython-%s%d"
         # see: http://stackoverflow.com/questions/11648440/python-does-not-detect-pyc-files
-        for f in os.listdir("dummy_package/__pycache__"):
+        pycache = os.path.join(dummy_package_path, "__pycache__")
+        for f in os.listdir(pycache):
             dst = f.replace('.cpython-%s%s' % sys.version_info[:2], "")
-            dst = os.path.join("dummy_package", dst)
-            shutil.copy(os.path.join("dummy_package/__pycache__", f), dst)
+            dst = os.path.join(dummy_package_path, dst)
+            shutil.copy(os.path.join(pycache, f), dst)
+    try:
+        yield path
+    finally:
+        shutil.rmtree(path)
 
 
-@cwd_at('test/test_evaluate')
-def test_pyc(Script):
+def test_pyc(pyc_project_path):
     """
     The list of completion must be greater than 2.
     """
-    try:
-        generate_pyc()
-        s = jedi.Script("from dummy_package import dummy; dummy.", path='blub.py')
-        assert len(s.completions()) >= 2
-    finally:
-        shutil.rmtree("dummy_package")
+    path = os.path.join(pyc_project_path, 'blub.py')
+    s = jedi.Script(
+        "from dummy_package import dummy; dummy.",
+        path=path,
+        environment=SameEnvironment())
+    assert len(s.completions()) >= 2

test/test_evaluate/test_sys_path.py

@@ -29,10 +29,8 @@ def test_paths_from_assignment(Script):
 def test_venv_and_pths(venv_path):
     pjoin = os.path.join
 
-    virtualenv = create_environment(venv_path)
-
     CUR_DIR = os.path.dirname(__file__)
-    site_pkg_path = pjoin(virtualenv.path, 'lib')
+    site_pkg_path = pjoin(venv_path, 'lib')
     if os.name == 'nt':
         site_pkg_path = pjoin(site_pkg_path, 'site-packages')
     else:
@@ -40,6 +38,7 @@ def test_venv_and_pths(venv_path):
     shutil.rmtree(site_pkg_path)
     shutil.copytree(pjoin(CUR_DIR, 'sample_venvs', 'pth_directory'), site_pkg_path)
 
+    virtualenv = create_environment(venv_path)
     venv_paths = virtualenv.get_sys_path()
 
     ETALON = [

test/test_parso_integration/test_parser_utils.py

@@ -85,4 +85,4 @@ def test_get_call_signature(code, call_signature):
         node = node.children[0]
     assert parser_utils.get_call_signature(node) == call_signature
 
-    assert parser_utils.get_doc_with_call_signature(node) == (call_signature + '\n\n')
+    assert parser_utils.get_doc_with_call_signature(node) == call_signature

test/test_settings.py

@@ -0,0 +1,19 @@
+import pytest
+
+from jedi import settings
+from jedi.evaluate.compiled import CompiledContextName
+
+
+@pytest.fixture()
+def auto_import_json(monkeypatch):
+    monkeypatch.setattr(settings, 'auto_import_modules', ['json'])
+
+
+def test_base_auto_import_modules(auto_import_json, Script):
+    loads, = Script('import json; json.loads').goto_definitions()
+    assert isinstance(loads._name, CompiledContextName)
+
+
+def test_auto_import_modules_imports(auto_import_json, Script):
+    main, = Script('from json import tool; tool.main').goto_definitions()
+    assert isinstance(main._name, CompiledContextName)

tox.ini

@@ -1,16 +1,10 @@
 [tox]
-envlist = py27, py33, py34, py35, py36
+envlist = py27, py34, py35, py36
 [testenv]
-deps =
-    pytest>=2.3.5, < 3.3
-    pytest-cache
-# docopt for sith doctests
-    docopt
-# coloroma for colored debug output
-    colorama
+extras = testing
 # Overwrite the parso version (only used sometimes).
-    git+https://github.com/davidhalter/parso.git
-#    -rrequirements.txt
+# deps =
+#     git+https://github.com/davidhalter/parso.git
 passenv = JEDI_TEST_ENVIRONMENT
 setenv =
 # https://github.com/tomchristie/django-rest-framework/issues/1957
@@ -19,44 +13,34 @@ setenv =
 # To test Jedi in different versions than the same Python version, set a
 # different test environment.
     env27: JEDI_TEST_ENVIRONMENT=27
-    env33: JEDI_TEST_ENVIRONMENT=33
     env34: JEDI_TEST_ENVIRONMENT=34
     env35: JEDI_TEST_ENVIRONMENT=35
     env36: JEDI_TEST_ENVIRONMENT=36
     env37: JEDI_TEST_ENVIRONMENT=37
 commands =
-    py.test {posargs:jedi test}
+    pytest {posargs:jedi test}
 [testenv:py27]
 deps =
 # for testing the typing module
     typing
 # numpydoc for typing scipy stack
     numpydoc
-    {[testenv]deps}
-[testenv:py33]
-deps =
-    typing
-    {[testenv]deps}
 [testenv:py34]
 deps =
     typing
     numpydoc
-    {[testenv]deps}
 [testenv:py35]
 deps =
     numpydoc
-    {[testenv]deps}
 [testenv:py36]
 deps =
     numpydoc
-    {[testenv]deps}
 [testenv:cov]
 deps =
     coverage
     numpydoc
-    {[testenv]deps}
 commands =
-    coverage run --source jedi -m py.test
+    coverage run --source jedi -m pytest
     coverage report
 [testenv:sith]
 commands =

travis_install.sh

@@ -8,10 +8,6 @@ set -e
 sudo chown root: /opt/python/3.6/bin/python
 sudo chown root: /opt/python/3.6.3/bin/python
 
-if [[ $JEDI_TEST_ENVIRONMENT == "33" ]]; then
-    VERSION=3.3
-    DOWNLOAD=1
-fi
 if [[ $JEDI_TEST_ENVIRONMENT == "35" ]]; then
     VERSION=3.5
     DOWNLOAD=1