From 73576b2a8b31a988df744c82089aa584dc68ac99 Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Fri, 17 Nov 2017 01:21:38 +0100 Subject: [PATCH] Progress when working with evaluators --- jedi/api/__init__.py | 2 +- jedi/evaluate/compiled/subprocess/__init__.py | 124 ++++++++++++++---- .../evaluate/compiled/subprocess/functions.py | 18 +++ jedi/evaluate/project.py | 4 +- test/test_api/test_environment.py | 10 ++ 5 files changed, 134 insertions(+), 24 deletions(-) diff --git a/jedi/api/__init__.py b/jedi/api/__init__.py index 30bec3b9..bfdaf528 100644 --- a/jedi/api/__init__.py +++ b/jedi/api/__init__.py @@ -121,7 +121,7 @@ class Script(object): compiled_subprocess = None else: if environment is None: - environment = DefaultEnvironment() + environment = DefaultEnvironment(self.path) compiled_subprocess = environment.get_subprocess() self._evaluator = Evaluator(self._grammar, project, compiled_subprocess) project.add_script_path(self.path) diff --git a/jedi/evaluate/compiled/subprocess/__init__.py b/jedi/evaluate/compiled/subprocess/__init__.py index 5d4d7dee..abf845fe 100644 --- a/jedi/evaluate/compiled/subprocess/__init__.py +++ b/jedi/evaluate/compiled/subprocess/__init__.py @@ -32,9 +32,8 @@ def get_subprocess(executable): return sub -def _get_function(evaluator, name): - function = getattr(functions, name) - return partial(function, evaluator) +def _get_function(name): + return getattr(functions, name) class EvaluatorSameProcess(object): @@ -47,20 +46,38 @@ class EvaluatorSameProcess(object): self._evaluator = evaluator def __getattr__(self, name): - return _get_function(self._evaluator, name) + return _get_function(name) class EvaluatorSubprocess(object): def __init__(self, evaluator, compiled_subprocess): + self._used = False self._evaluator_weakref = weakref.ref(evaluator) - self._evaluator_id = () + self._evaluator_id = id(evaluator) self._compiled_subprocess = compiled_subprocess def __getattr__(self, name): - return _get_function(self._evaluator_weakref(), name) + func = _get_function(name) + + def wrapper(*args, **kwargs): + self._used = True + + result = self._compiled_subprocess.run( + self._evaluator_weakref(), + func, + *args, + **kwargs + ) + if isinstance(result, CompiledHandle): + result.add_subprocess(self._compiled_subprocess) + + return result + + return wrapper def __del__(self): - self.delete_evaluator(self._evaluator_weakref()) + if self._used: + self._compiled_subprocess.delete_evaluator(self._evaluator_id) class _Subprocess(object): @@ -74,10 +91,11 @@ class _Subprocess(object): self._args, stdin=subprocess.PIPE, stdout=subprocess.PIPE, - #stderr=subprocess.PIPE + # stderr=subprocess.PIPE ) - def _send(self, *data): + def _send(self, evaluator_id, function, args=(), kwargs={}): + data = evaluator_id, function, args, kwargs pickle.dump(data, self._process.stdin, protocol=_PICKLE_PROTOCOL) self._process.stdin.flush() return pickle.load(self._process.stdout) @@ -116,27 +134,24 @@ class Listener(): self._evaluators = {} def _get_evaluator(self, function, evaluator_id): - from jedi.evaluate import Evaluator - - if function is None: - # If the function is None, this is the hint to delete the - # evaluator. - del self._evaluators[evaluator_id] - return + from jedi.evaluate import Evaluator, project try: - evaluator = self.evaluators[evaluator_id] + evaluator, handles = self._evaluators[evaluator_id] except KeyError: - evaluator = Evaluator(None, None) - self.evaluators[evaluator_id] = evaluator - return evaluator + evaluator = Evaluator(None, project=project.Project()) + handles = Handles() + self._evaluators[evaluator_id] = evaluator, handles + return evaluator, handles def _run(self, evaluator_id, function, args, kwargs): if evaluator_id is None: return function(*args, **kwargs) + elif function is None: + del self._evaluators[evaluator_id] else: - evaluator = self._get_evaluator(evaluator_id) - return function(evaluator, *args, **kwargs) + evaluator, handles = self._get_evaluator(function, evaluator_id) + return function(evaluator, handles, *args, **kwargs) def listen(self): stdout = sys.stdout @@ -155,3 +170,68 @@ class Listener(): result = self._run(*payload) pickle.dump(result, stdout, protocol=_PICKLE_PROTOCOL) stdout.flush() + + +''' +class ModifiedUnpickler(pickle._Unpickler): + dispatch = pickle._Unpickler.dispatch.copy() + + def __init__(self, subprocess, *args, **kwargs): + super(ModifiedUnpickler, self).__init__(*args, **kwargs) + self._subprocess = subprocess + + def load_newobj(self): + """ + Just a copy of the builtin plus the on_new_obj hook. + """ + args = self.stack.pop() + cls = self.stack.pop() + obj = cls.__new__(cls, *args) + self.append(self.on_new_obj(obj)) + + def on_new_obj(self, obj): + if isinstance(obj, CompiledHandle): + obj.add_subprocess(self._subprocess) + dispatch[pickle.NEWOBJ[0]] = load_newobj +''' + + +class Handles(object): + def __init__(self): + self._handles = {} + + def create(self, obj): + handle = self._handles[id(obj)] = CompiledHandle(obj) + return handle + + def get_compiled_object(self, id_): + return self._handles[id_].compiled_object + + +class CompiledHandle(object): + def __init__(self, compiled_object): + self._compiled_object = compiled_object + self._id = id(compiled_object) + + def add_subprocess(self, subprocess): + self._subprocess = subprocess + + def __getstate__(self): + return self._id + + def __setstate__(self, state): + self._id = state + + def __getattr__(self, name): + from jedi.evaluate import compiled + attr = getattr(compiled.CompiledObject, name) + if isinstance(attr, property): + return self._subprocess.get_compiled_property(self._id, name) + elif isinstance(attr, compiled.CheckAttribute): + # It might raise an AttributeError, however we're interested in the + # function return value. + self._subprocess.get_compiled_property(self._id, name) + + def _compiled_method(*args, **kwargs): + return self._subprocess.get_compiled_method_return(self._id, name, *args, **kwargs) + return self._compiled_method diff --git a/jedi/evaluate/compiled/subprocess/functions.py b/jedi/evaluate/compiled/subprocess/functions.py index 49417235..caaa0bd7 100644 --- a/jedi/evaluate/compiled/subprocess/functions.py +++ b/jedi/evaluate/compiled/subprocess/functions.py @@ -1,5 +1,23 @@ import sys +from jedi.evaluate import compiled def get_sys_path(): return sys.path + + +def import_module(evaluator, handles, path=None, name=None): + compiled_object = compiled.load_module(evaluator, path=path, name=name) + if compiled_object is None: + return None + return handles.create(compiled_object) + + +def get_compiled_property(evaluator, handles, id, attribute): + compiled_object = handles.get_compiled_object(id) + return getattr(compiled_object, attribute) + + +def get_compiled_method_return(evaluator, handles, id, attribute, *args, **kwargs): + compiled_object = handles.get_compiled_object(id) + return getattr(compiled_object, attribute)(*args, **kwargs) diff --git a/jedi/evaluate/project.py b/jedi/evaluate/project.py index fe3fb5d7..6a3df7e9 100644 --- a/jedi/evaluate/project.py +++ b/jedi/evaluate/project.py @@ -7,8 +7,10 @@ from jedi.cache import memoize_method class Project(object): def __init__(self, sys_path=None): + self._script_path = None + if sys_path is not None: - self._sys_path = sys_path + self._base_sys_path = sys_path venv = os.getenv('VIRTUAL_ENV') if venv: diff --git a/test/test_api/test_environment.py b/test/test_api/test_environment.py index 2a37aff5..61a943c0 100644 --- a/test/test_api/test_environment.py +++ b/test/test_api/test_environment.py @@ -1,5 +1,6 @@ import pytest +import jedi from jedi._compatibility import py_version from jedi.api.virtualenv import Environment, DefaultEnvironment, NoVirtualEnv @@ -24,3 +25,12 @@ def test_versions(version): sys_path = env.get_sys_path() assert any(executable in p for p in sys_path) + + +@pytest.fixture +def evaluator(): + return jedi.Script('')._evaluator + + +def test_import_module(evaluator): + evaluator.compiled_subprocess.import_module(name='math')