diff --git a/jedi/_compatibility.py b/jedi/_compatibility.py index 52a20fe2..64469c8a 100644 --- a/jedi/_compatibility.py +++ b/jedi/_compatibility.py @@ -289,3 +289,9 @@ def utf8_repr(func): return func else: return wrapper + + +if is_py3: + import queue +else: + import Queue as queue diff --git a/jedi/evaluate/__init__.py b/jedi/evaluate/__init__.py index f07abf88..f015ac40 100644 --- a/jedi/evaluate/__init__.py +++ b/jedi/evaluate/__init__.py @@ -107,9 +107,7 @@ class Evaluator(object): if compiled_sub_process is None: self.compiled_subprocess = EvaluatorSameProcess(self) else: - self.compiled_subprocess = EvaluatorSameProcess(self) - #self.compiled_subprocess = EvaluatorSubprocess(self, compiled_sub_process) - + self.compiled_subprocess = EvaluatorSubprocess(self, compiled_sub_process) self.reset_recursion_limitations() diff --git a/jedi/evaluate/compiled/access.py b/jedi/evaluate/compiled/access.py index 69066e6e..82d143c9 100644 --- a/jedi/evaluate/compiled/access.py +++ b/jedi/evaluate/compiled/access.py @@ -274,9 +274,9 @@ class DirectObjectAccess(object): return None def get_safe_value(self): - if type(self._obj) in (float, int, str, unicode, slice, type(Ellipsis)): + if type(self._obj) in (bool, bytes, float, int, str, unicode, slice, type(Ellipsis)): return self._obj - raise ValueError + raise ValueError("Object is type %s and not simple" % type(self._obj)) def get_api_type(self): obj = self._obj diff --git a/jedi/evaluate/compiled/subprocess/__init__.py b/jedi/evaluate/compiled/subprocess/__init__.py index b59e39ee..07e0d279 100644 --- a/jedi/evaluate/compiled/subprocess/__init__.py +++ b/jedi/evaluate/compiled/subprocess/__init__.py @@ -14,6 +14,7 @@ import weakref import pickle from functools import partial +from jedi._compatibility import queue from jedi.cache import memoize_method from jedi.evaluate.compiled.subprocess import functions from jedi.evaluate.compiled.access import DirectObjectAccess @@ -68,7 +69,7 @@ class EvaluatorSameProcess(_EvaluatorProcess): class EvaluatorSubprocess(_EvaluatorProcess): def __init__(self, evaluator, compiled_subprocess): - super(EvaluatorSubprocess).__init__(evaluator) + super(EvaluatorSubprocess, self).__init__(evaluator) self._used = False self._compiled_subprocess = compiled_subprocess @@ -83,7 +84,7 @@ class EvaluatorSubprocess(_EvaluatorProcess): func, args=args, kwargs=kwargs, - unpickler=lambda stdout: ModifiedUnpickler(self, stdout).load() + unpickler=lambda stdout: _ModifiedMasterUnpickler(self, stdout).load() ) if isinstance(result, AccessHandle): result.add_subprocess(self) @@ -136,8 +137,18 @@ class _CompiledSubprocess(_Subprocess): os.path.dirname(os.path.dirname(parso_path)) ) ) + self._evaluator_deletion_queue = queue.deque() def run(self, evaluator, function, args=(), kwargs={}, unpickler=None): + # Delete old evaluators. + while True: + try: + evaluator_id = self._evaluator_deletion_queue.pop() + except IndexError: + break + else: + self._send(evaluator_id, None) + assert callable(function) return self._send(id(evaluator), function, args, kwargs, unpickler=unpickler) @@ -145,13 +156,22 @@ class _CompiledSubprocess(_Subprocess): return self._send(None, functions.get_sys_path, (), {}) def delete_evaluator(self, evaluator_id): + """ + Currently we are not deleting evalutors instantly. They only get + deleted once the subprocess is used again. It would probably a better + solution to move all of this into a thread. However, the memory usage + of a single evaluator shouldn't be that high. + """ # With an argument - the evaluator gets deleted. - self._send(evaluator_id, None) + self._evaluator_deletion_queue.append(evaluator_id) class Listener(): def __init__(self): self._evaluators = {} + # TODO refactor so we don't need to process anymore just handle + # controlling. + self._process = _EvaluatorProcess(Listener) def _get_evaluator(self, function, evaluator_id): from jedi.evaluate import Evaluator, project @@ -170,6 +190,16 @@ class Listener(): del self._evaluators[evaluator_id] else: evaluator = self._get_evaluator(function, evaluator_id) + + # Exchange all handles + args = list(args) + for i, arg in enumerate(args): + if isinstance(arg, AccessHandle): + args[i] = evaluator.compiled_subprocess.get_access_handle(arg.id) + for key, value in kwargs.items(): + if isinstance(value, AccessHandle): + kwargs[key] = evaluator.compiled_subprocess.get_access_handle(value.id) + return function(evaluator, *args, **kwargs) def listen(self): @@ -181,7 +211,7 @@ class Listener(): while True: try: - payload = pickle.load(stdin) + payload = pickle.load(file=stdin) except EOFError: # It looks like the parent process closed. Don't make a big fuss # here and just exit. @@ -191,54 +221,48 @@ class Listener(): except Exception as e: result = True, e - print(result, payload, file=sys.stderr) - - ModifiedPickler(file=stdout, protocol=_PICKLE_PROTOCOL).dump(result) + pickle.dump(result, file=stdout, protocol=_PICKLE_PROTOCOL) stdout.flush() -class ModifiedUnpickler(pickle._Unpickler): +class _ModifiedUnpickler(pickle._Unpickler): dispatch = pickle._Unpickler.dispatch.copy() def __init__(self, subprocess, *args, **kwargs): - super(ModifiedUnpickler, self).__init__(*args, **kwargs) + super(_ModifiedUnpickler, self).__init__(*args, **kwargs) self._subprocess = subprocess def load_newobj(self): - super(ModifiedUnpickler, self).load_newobj() + super(_ModifiedUnpickler, self).load_newobj() tos = self.stack[-1] - print('pop', tos, file=sys.stderr) if isinstance(tos, AccessHandle): - tos.add_subprocess(self._subprocess) + self.stack[-1] = self.get_access_handle(tos) dispatch[pickle.NEWOBJ[0]] = load_newobj -class ModifiedPickler(pickle._Pickler): - def __init__(self, *args, **kwargs): - super(ModifiedPickler, self).__init__(*args, **kwargs) - - def save(self, obj, *args, **kwargs): - print('s', obj, args, kwargs, file=sys.stderr) - return super(ModifiedPickler, self).save(obj, *args, **kwargs) +class _ModifiedMasterUnpickler(_ModifiedUnpickler): + def get_access_handle(self, access_handle): + access_handle.add_subprocess(self._subprocess) + return access_handle class AccessHandle(object): def __init__(self, subprocess, access, id_): self.access = access self._subprocess = subprocess - self._id = id_ + self.id = id_ def add_subprocess(self, subprocess): self._subprocess = subprocess def __getstate__(self): - return self._id + return self.id def __setstate__(self, state): - self._id = state + self.id = state def __getattr__(self, name): - print('getattr', name, file=sys.stderr) + #print('getattr', name, file=sys.stderr) def compiled_method(*args, **kwargs): - return self._subprocess.get_compiled_method_return(self._id, name, *args, **kwargs) + return self._subprocess.get_compiled_method_return(self.id, name, *args, **kwargs) return compiled_method diff --git a/test/test_evaluate/test_literals.py b/test/test_evaluate/test_literals.py index 678c27a8..491ca520 100644 --- a/test/test_evaluate/test_literals.py +++ b/test/test_evaluate/test_literals.py @@ -6,7 +6,7 @@ from jedi._compatibility import py_version, unicode def _eval_literal(code): def_, = jedi.Script(code).goto_definitions() - return def_._name._context.access_handle.access._obj + return def_._name._context.get_safe_value() @pytest.mark.skipif('sys.version_info[:2] < (3, 6)') diff --git a/test/test_evaluate/test_precedence.py b/test/test_evaluate/test_precedence.py index 0d98d62c..02dbbecb 100644 --- a/test/test_evaluate/test_precedence.py +++ b/test/test_evaluate/test_precedence.py @@ -14,4 +14,4 @@ def test_equals(source): script = Script(source) node = script._get_module_node().children[0] first, = script._get_module().eval_node(node) - assert isinstance(first, CompiledObject) and first.access_handle.access._obj is True + assert isinstance(first, CompiledObject) and first.get_safe_value() is True