Environments are now always created on request

The issue was that if something changed about the environment (e.g. version
switch) or sys.path change, re-creating the environment was possible, but did
not involve the change. The environments have now a __del__ function that
deletes the subprocess after every time an Environment is garbage collected.

jedi/api/environment.py

@@ -10,7 +10,7 @@ from collections import namedtuple
 
 from jedi._compatibility import highest_pickle_protocol, which
 from jedi.cache import memoize_method, time_cache
-from jedi.evaluate.compiled.subprocess import get_subprocess, \
+from jedi.evaluate.compiled.subprocess import CompiledSubprocess, \
     EvaluatorSameProcess, EvaluatorSubprocess
 
 import parso
@@ -58,16 +58,28 @@ class Environment(_BaseEnvironment):
     should not create it directly. Please use create_environment or the other
     functions instead. It is then returned by that function.
     """
+    _subprocess = None
+
     def __init__(self, executable):
+        self._start_executable = executable
+        # Initialize the environment
+        self._get_subprocess()
+
+    def _get_subprocess(self):
+        if self._subprocess is not None and not self._subprocess.is_crashed:
+            return self._subprocess
+
         try:
-            self._subprocess = get_subprocess(executable)
+            self._subprocess = CompiledSubprocess(self._start_executable)
             info = self._subprocess._send(None, _get_info)
         except Exception as exc:
             raise InvalidPythonEnvironment(
                 "Could not get version information for %r: %r" % (
-                    executable,
+                    self._start_executable,
                     exc))
 
+        # Since it could change and might not be the same(?) as the one given,
+        # set it here.
         self.executable = info[0]
         """
         The Python executable, matches ``sys.executable``.
@@ -82,15 +94,17 @@ class Environment(_BaseEnvironment):
         Python version.
         """
 
-        # Adjust pickle protocol according to host and client version.
-        self._subprocess._pickle_protocol = highest_pickle_protocol([
-            sys.version_info, self.version_info])
-
         # py2 sends bytes via pickle apparently?!
         if self.version_info.major == 2:
             self.executable = self.executable.decode()
             self.path = self.path.decode()
 
+        # Adjust pickle protocol according to host and client version.
+        self._subprocess._pickle_protocol = highest_pickle_protocol([
+            sys.version_info, self.version_info])
+
+        return self._subprocess
+
     def __repr__(self):
         version = '.'.join(str(i) for i in self.version_info)
         return '<%s: %s in %s>' % (self.__class__.__name__, version, self.path)
@@ -98,9 +112,6 @@ class Environment(_BaseEnvironment):
     def get_evaluator_subprocess(self, evaluator):
         return EvaluatorSubprocess(evaluator, self._get_subprocess())
 
-    def _get_subprocess(self):
-        return get_subprocess(self.executable)
-
     @memoize_method
     def get_sys_path(self):
         """
@@ -119,7 +130,7 @@ class Environment(_BaseEnvironment):
 
 class SameEnvironment(Environment):
     def __init__(self):
-        self.executable = sys.executable
+        self._start_executable = self.executable = sys.executable
         self.path = sys.prefix
         self.version_info = _VersionInfo(*sys.version_info[:3])
 

jedi/evaluate/compiled/subprocess/__init__.py

@@ -24,19 +24,10 @@ from jedi.evaluate.compiled.access import DirectObjectAccess, AccessPath, \
     SignatureParam
 from jedi.api.exceptions import InternalError
 
-_subprocesses = {}
 
 _MAIN_PATH = os.path.join(os.path.dirname(__file__), '__main__.py')
 
 
-def get_subprocess(executable):
-    try:
-        return _subprocesses[executable]
-    except KeyError:
-        sub = _subprocesses[executable] = _CompiledSubprocess(executable)
-        return sub
-
-
 def _get_function(name):
     return getattr(functions, name)
 
@@ -118,12 +109,12 @@ class EvaluatorSubprocess(_EvaluatorProcess):
         return obj
 
     def __del__(self):
-        if self._used:
+        if self._used and not self._compiled_subprocess.is_crashed:
             self._compiled_subprocess.delete_evaluator(self._evaluator_id)
 
 
-class _CompiledSubprocess(object):
+class CompiledSubprocess(object):
-    _crashed = False
+    is_crashed = False
     # Start with 2, gets set after _get_info.
     _pickle_protocol = 2
 
@@ -133,10 +124,11 @@ class _CompiledSubprocess(object):
 
     def __repr__(self):
         pid = os.getpid()
-        return '<_CompiledSubprocess _executable=%r, _pickle_protocol=%r, _crashed=%r, pid=%r>' % (
+        return '<%s _executable=%r, _pickle_protocol=%r, is_crashed=%r, pid=%r>' % (
+            self.__class__.__name__,
             self._executable,
             self._pickle_protocol,
-            self._crashed,
+            self.is_crashed,
             pid,
         )
 
@@ -176,24 +168,22 @@ class _CompiledSubprocess(object):
     def get_sys_path(self):
         return self._send(None, functions.get_sys_path, (), {})
 
-    def kill(self):
+    def _kill(self):
-        self._crashed = True
+        self.is_crashed = True
-        try:
+        if subprocess.signal is None:
-            subprocess = _subprocesses[self._executable]
+            # If the Python process is terminating, sometimes it will remove
-        except KeyError:
+            # the signal module before a lot of other things, so check for it
-            # Fine it was already removed from the cache.
+            # and don't do anything, because the process is killed anyways.
-            pass
+            return
-        else:
-            # In the `!=` case there is already a new subprocess in place
-            # and we don't need to do anything here anymore.
-            if subprocess == self:
-                del _subprocesses[self._executable]
-
         self._process.kill()
         self._process.wait()
 
+    def __del__(self):
+        if not self.is_crashed:
+            self._kill()
+
     def _send(self, evaluator_id, function, args=(), kwargs={}):
-        if self._crashed:
+        if self.is_crashed:
             raise InternalError("The subprocess %s has crashed." % self._executable)
 
         if not is_py3:
@@ -210,7 +200,7 @@ class _CompiledSubprocess(object):
             if e.errno not in (errno.EPIPE, errno.EINVAL):
                 # Not a broken pipe
                 raise
-            self.kill()
+            self._kill()
             raise InternalError("The subprocess %s was killed. Maybe out of memory?"
                                 % self._executable)
 
@@ -221,7 +211,7 @@ class _CompiledSubprocess(object):
                 stderr = self._process.stderr.read()
             except Exception as exc:
                 stderr = '<empty/not available (%r)>' % exc
-            self.kill()
+            self._kill()
             raise InternalError(
                 "The subprocess %s has crashed (%r, stderr=%s)." % (
                     self._executable,