Get the subprocess mostly working

jedi/_compatibility.py

@@ -289,3 +289,9 @@ def utf8_repr(func):
         return func
     else:
         return wrapper
+
+
+if is_py3:
+    import queue
+else:
+    import Queue as queue

jedi/evaluate/__init__.py

@@ -107,9 +107,7 @@ class Evaluator(object):
         if compiled_sub_process is None:
             self.compiled_subprocess = EvaluatorSameProcess(self)
         else:
-            self.compiled_subprocess = EvaluatorSameProcess(self)
+            self.compiled_subprocess = EvaluatorSubprocess(self, compiled_sub_process)
-            #self.compiled_subprocess = EvaluatorSubprocess(self, compiled_sub_process)
-
 
         self.reset_recursion_limitations()
 

jedi/evaluate/compiled/access.py

@@ -274,9 +274,9 @@ class DirectObjectAccess(object):
             return None
 
     def get_safe_value(self):
-        if type(self._obj) in (float, int, str, unicode, slice, type(Ellipsis)):
+        if type(self._obj) in (bool, bytes, float, int, str, unicode, slice, type(Ellipsis)):
             return self._obj
-        raise ValueError
+        raise ValueError("Object is type %s and not simple" % type(self._obj))
 
     def get_api_type(self):
         obj = self._obj

jedi/evaluate/compiled/subprocess/__init__.py

@@ -14,6 +14,7 @@ import weakref
 import pickle
 from functools import partial
 
+from jedi._compatibility import queue
 from jedi.cache import memoize_method
 from jedi.evaluate.compiled.subprocess import functions
 from jedi.evaluate.compiled.access import DirectObjectAccess
@@ -68,7 +69,7 @@ class EvaluatorSameProcess(_EvaluatorProcess):
 
 class EvaluatorSubprocess(_EvaluatorProcess):
     def __init__(self, evaluator, compiled_subprocess):
-        super(EvaluatorSubprocess).__init__(evaluator)
+        super(EvaluatorSubprocess, self).__init__(evaluator)
         self._used = False
         self._compiled_subprocess = compiled_subprocess
 
@@ -83,7 +84,7 @@ class EvaluatorSubprocess(_EvaluatorProcess):
                 func,
                 args=args,
                 kwargs=kwargs,
-                unpickler=lambda stdout: ModifiedUnpickler(self, stdout).load()
+                unpickler=lambda stdout: _ModifiedMasterUnpickler(self, stdout).load()
             )
             if isinstance(result, AccessHandle):
                 result.add_subprocess(self)
@@ -136,8 +137,18 @@ class _CompiledSubprocess(_Subprocess):
              os.path.dirname(os.path.dirname(parso_path))
              )
         )
+        self._evaluator_deletion_queue = queue.deque()
 
     def run(self, evaluator, function, args=(), kwargs={}, unpickler=None):
+        # Delete old evaluators.
+        while True:
+            try:
+                evaluator_id = self._evaluator_deletion_queue.pop()
+            except IndexError:
+                break
+            else:
+                self._send(evaluator_id, None)
+
         assert callable(function)
         return self._send(id(evaluator), function, args, kwargs, unpickler=unpickler)
 
@@ -145,13 +156,22 @@ class _CompiledSubprocess(_Subprocess):
         return self._send(None, functions.get_sys_path, (), {})
 
     def delete_evaluator(self, evaluator_id):
+        """
+        Currently we are not deleting evalutors instantly. They only get
+        deleted once the subprocess is used again. It would probably a better
+        solution to move all of this into a thread. However, the memory usage
+        of a single evaluator shouldn't be that high.
+        """
         # With an argument - the evaluator gets deleted.
-        self._send(evaluator_id, None)
+        self._evaluator_deletion_queue.append(evaluator_id)
 
 
 class Listener():
     def __init__(self):
         self._evaluators = {}
+        # TODO refactor so we don't need to process anymore just handle
+        # controlling.
+        self._process = _EvaluatorProcess(Listener)
 
     def _get_evaluator(self, function, evaluator_id):
         from jedi.evaluate import Evaluator, project
@@ -170,6 +190,16 @@ class Listener():
             del self._evaluators[evaluator_id]
         else:
             evaluator = self._get_evaluator(function, evaluator_id)
+
+            # Exchange all handles
+            args = list(args)
+            for i, arg in enumerate(args):
+                if isinstance(arg, AccessHandle):
+                    args[i] = evaluator.compiled_subprocess.get_access_handle(arg.id)
+            for key, value in kwargs.items():
+                if isinstance(value, AccessHandle):
+                    kwargs[key] = evaluator.compiled_subprocess.get_access_handle(value.id)
+
             return function(evaluator, *args, **kwargs)
 
     def listen(self):
@@ -181,7 +211,7 @@ class Listener():
 
         while True:
             try:
-                payload = pickle.load(stdin)
+                payload = pickle.load(file=stdin)
             except EOFError:
                 # It looks like the parent process closed. Don't make a big fuss
                 # here and just exit.
@@ -191,54 +221,48 @@ class Listener():
             except Exception as e:
                 result = True, e
 
-            print(result, payload, file=sys.stderr)
+            pickle.dump(result, file=stdout, protocol=_PICKLE_PROTOCOL)
-
-            ModifiedPickler(file=stdout, protocol=_PICKLE_PROTOCOL).dump(result)
             stdout.flush()
 
 
-class ModifiedUnpickler(pickle._Unpickler):
+class _ModifiedUnpickler(pickle._Unpickler):
     dispatch = pickle._Unpickler.dispatch.copy()
 
     def __init__(self, subprocess, *args, **kwargs):
-        super(ModifiedUnpickler, self).__init__(*args, **kwargs)
+        super(_ModifiedUnpickler, self).__init__(*args, **kwargs)
         self._subprocess = subprocess
 
     def load_newobj(self):
-        super(ModifiedUnpickler, self).load_newobj()
+        super(_ModifiedUnpickler, self).load_newobj()
         tos = self.stack[-1]
-        print('pop', tos, file=sys.stderr)
         if isinstance(tos, AccessHandle):
-            tos.add_subprocess(self._subprocess)
+            self.stack[-1] = self.get_access_handle(tos)
     dispatch[pickle.NEWOBJ[0]] = load_newobj
 
 
-class ModifiedPickler(pickle._Pickler):
+class _ModifiedMasterUnpickler(_ModifiedUnpickler):
-    def __init__(self, *args, **kwargs):
+    def get_access_handle(self, access_handle):
-        super(ModifiedPickler, self).__init__(*args, **kwargs)
+        access_handle.add_subprocess(self._subprocess)
-
+        return access_handle
-    def save(self, obj, *args, **kwargs):
-        print('s', obj, args, kwargs, file=sys.stderr)
-        return super(ModifiedPickler, self).save(obj, *args, **kwargs)
 
 
 class AccessHandle(object):
     def __init__(self, subprocess, access, id_):
         self.access = access
         self._subprocess = subprocess
-        self._id = id_
+        self.id = id_
 
     def add_subprocess(self, subprocess):
         self._subprocess = subprocess
 
     def __getstate__(self):
-        return self._id
+        return self.id
 
     def __setstate__(self, state):
-        self._id = state
+        self.id = state
 
     def __getattr__(self, name):
-        print('getattr', name, file=sys.stderr)
+        #print('getattr', name, file=sys.stderr)
         def compiled_method(*args, **kwargs):
-            return self._subprocess.get_compiled_method_return(self._id, name, *args, **kwargs)
+            return self._subprocess.get_compiled_method_return(self.id, name, *args, **kwargs)
         return compiled_method

test/test_evaluate/test_literals.py

@@ -6,7 +6,7 @@ from jedi._compatibility import py_version, unicode
 
 def _eval_literal(code):
     def_, = jedi.Script(code).goto_definitions()
-    return def_._name._context.access_handle.access._obj
+    return def_._name._context.get_safe_value()
 
 
 @pytest.mark.skipif('sys.version_info[:2] < (3, 6)')

test/test_evaluate/test_precedence.py

@@ -14,4 +14,4 @@ def test_equals(source):
     script = Script(source)
     node = script._get_module_node().children[0]
     first, = script._get_module().eval_node(node)
-    assert isinstance(first, CompiledObject) and first.access_handle.access._obj is True
+    assert isinstance(first, CompiledObject) and first.get_safe_value() is True