Subprocess progress

Also add an enviornment variable to Script

jedi/api/__init__.py

@@ -36,6 +36,7 @@ from jedi.evaluate.syntax_tree import tree_name_to_contexts
 from jedi.evaluate.context import ModuleContext
 from jedi.evaluate.context.module import ModuleName
 from jedi.evaluate.context.iterable import unpack_tuple_to_dict
+from jedi.evaluate.compiled.subprocess import get_subprocess
 
 # Jedi uses lots and lots of recursion. By setting this a little bit higher, we
 # can remove some "maximum recursion depth" errors.
@@ -79,10 +80,12 @@ class Script(object):
     :type encoding: str
     :param sys_path: ``sys.path`` to use during analysis of the script
     :type sys_path: list
+    :param environment: TODO
+    :type sys_path: Environment
 
     """
     def __init__(self, source=None, line=None, column=None, path=None,
-                 encoding='utf-8', sys_path=None):
+                 encoding='utf-8', sys_path=None, environment=None):
         self._orig_path = path
         # An empty path (also empty string) should always result in no path.
         self.path = os.path.abspath(path) if path else None
@@ -112,10 +115,24 @@ class Script(object):
         # Load the Python grammar of the current interpreter.
         self._grammar = parso.load_grammar()
         project = Project(sys_path=sys_path)
-        self._evaluator = Evaluator(self._grammar, project)
+        if isinstance(self, Interpreter):
+            # It's not possible to use a subprocess for the interpreter.
+            self._compiled_subprocess = None
+        else:
+            if environment is None:
+                executable = sys.executable
+            else:
+                executable = environment.executable
+            self._compiled_subprocess = get_subprocess(executable)
+        self._evaluator = Evaluator(self._grammar, project, self._compiled_subprocess)
         project.add_script_path(self.path)
         debug.speed('init')
 
+    def __del__(self):
+        if self._compiled_subprocess is not None:
+            self._compiled_subprocess.delete_evaluator(evaluator)
+        self._evaluator.cleanup_evaluator
+
     @cache.memoize_method
     def _get_module_node(self):
         return self._grammar.parse(
@@ -356,6 +373,9 @@ class Interpreter(Script):
         except Exception:
             raise TypeError("namespaces must be a non-empty list of dicts.")
 
+        if 'environment' in kwds:
+            raise TypeError("Environments are not allowed when using an interpreter.")
+
         super(Interpreter, self).__init__(source, **kwds)
         self.namespaces = namespaces
 

jedi/evaluate/__init__.py

@@ -83,10 +83,11 @@ from jedi.evaluate.context import ClassContext, FunctionContext, \
 from jedi.evaluate.context.iterable import CompForContext
 from jedi.evaluate.syntax_tree import eval_trailer, eval_expr_stmt, \
     eval_node, check_tuple_assignments
+from jedi.evaluate.compiled.subprocess import EvaluatorSubprocess, EvaluatorSameProcess
 
 
 class Evaluator(object):
-    def __init__(self, grammar, project):
+    def __init__(self, grammar, project, compiled_sub_process=None):
         self.grammar = grammar
         self.latest_grammar = parso.load_grammar(version='3.6')
         self.memoize_cache = {}  # for memoize decorators
@@ -102,6 +103,12 @@ class Evaluator(object):
         self.project = project
         project.add_evaluator(self)
 
+        if compiled_sub_process is None:
+            self.compiled_subprocess = EvaluatorSameProcess(self)
+        else:
+            self.compiled_subprocess = EvaluatorSubprocess(self, compiled_sub_process)
+
+
         self.reset_recursion_limitations()
 
         # Constants

jedi/evaluate/compiled/subprocess/__init__.py

@@ -9,21 +9,69 @@ goals:
 
 import sys
 import subprocess
+import weakref
 import pickle
+from functools import partial
+
+from jedi.cache import memoize_method
+from jedi.evaluate.compiled.subprocess import commands
 
 _PICKLE_PROTOCOL = 2
 
+_subprocesses = {}
 
-class _SubProcess(object):
+
+def get_subprocess(executable):
+    try:
+        return _subprocesses[executable]
+    except KeyError:
+        sub = _subprocesses[executable] = _CompiledSubprocess(executable)
+        return sub
+
+
+class EvaluatorSameProcess(object):
+    """
+    Basically just an easy access to functions.py. It has the same API
+    as EvaluatorSubprocess and does the same thing without using a subprocess.
+    This is necessary for the Interpreter process.
+    """
+    def __init__(self, evaluator):
+        self._evaluator = evaluator
+
+    def __getattr__(self):
+        function = getattr(commands, name)
+        return partial(function, self._evaluator)
+
+
+class EvaluatorSubprocess(object):
+    def __init__(self, evaluator, compiled_subprocess):
+        self._evaluator_weakref = weakref.ref(evaluator)
+        self._evaluator_id = ()
+        self._compiled_subprocess = compiled_subprocess
+
+    def __getattr__(self, name):
+        function = getattr(commands, name)
+        return partial(function, self._evaluator_weakref())
+
+    def __del__(self):
+        self.delete_evaluator(self._evaluator_weakref()
+
+
+class _Subprocess(object):
     def __init__(self, args):
-        self._process = subprocess.Popen(
+        self._args = args
-            args,
+
+    @property
+    @memoize_method
+    def _process(self):
+        return subprocess.Popen(
+            self._args,
             stdin=subprocess.PIPE,
             stdout=subprocess.PIPE,
             #stderr=subprocess.PIPE
         )
 
-    def _send(self, data):
+    def _send(self, *data):
         pickle.dump(data, self._process.stdin, protocol=_PICKLE_PROTOCOL)
         self._process.stdin.flush()
         return pickle.load(self._process.stdout)
@@ -35,30 +83,56 @@ class _SubProcess(object):
         self._process.kill()
 
 
-class CompiledSubProcess(object):
+class _CompiledSubprocess(_Subprocess):
     def __init__(self, executable):
-        super(CompiledSubProcess, self).__init__(
+        super(_CompiledSubprocess, self).__init__(
             (executable, '-m', 'jedi.evaluate.compiled.subprocess')
         )
 
-    def command(self, command):
+    def run(self, evaluator, function, *args, **kwargs):
-        return self._send()
+        assert callable(function)
+        return self._send(id(evaluator), function, args, kwargs)
+
+    def delete_evaluator(self, evaluator_id):
+        # With an argument - the evaluator gets deleted.
+        self._send(evaluator_id, None)
 
 
-def listen():
+class Listener():
-    stdout = sys.stdout
+    def __init__(self):
-    stdin = sys.stdin
+        self._evaluators = {}
-    if sys.version_info[0] > 2:
+
-        stdout = stdout.buffer
+    def _run(self, evaluator_id, function, args, kwargs):
-        stdin = stdin.buffer
+        from jedi.evaluate import Evaluator
+
+        if function is None:
+            # If the function is None, this is the hint to delete the
+            # evaluator.
+            del self._evaluators[evaluator_id]
+            return
 
-    while True:
         try:
-            result = pickle.load(stdin)
+            evaluator = self.evaluators[evaluator_id]
-        except EOFError:
+        except KeyError:
-            # It looks like the parent process closed. Don't make a big fuss
+            evaluator = Evaluator(None, None)
-            # here and just exit.
+            self.evaluators[evaluator_id] = evaluator
-            exit(1)
+
-        result += 1
+        return function(evaluator, *args, **kwargs)
-        pickle.dump(result, stdout, protocol=_PICKLE_PROTOCOL)
+
-        stdout.flush()
+    def listen(self):
+        stdout = sys.stdout
+        stdin = sys.stdin
+        if sys.version_info[0] > 2:
+            stdout = stdout.buffer
+            stdin = stdin.buffer
+
+        while True:
+            try:
+                payload = pickle.load(stdin)
+            except EOFError:
+                # It looks like the parent process closed. Don't make a big fuss
+                # here and just exit.
+                exit(1)
+            result = self._run(*payload)
+            pickle.dump(result, stdout, protocol=_PICKLE_PROTOCOL)
+            stdout.flush()

jedi/evaluate/project.py

@@ -3,10 +3,12 @@ import sys
 
 from jedi.evaluate.sys_path import get_venv_path, detect_additional_paths
 from jedi.cache import memoize_method
+from jedi.evaluate.compiled.subprocess import CompiledSubProcess
 
 
 class Project(object):
     def __init__(self, sys_path=None):
+        self._compiled_subprocess = CompiledSubProcess()
         if sys_path is not None:
             self._sys_path = sys_path