From 46b81dfa6df063dd4cc4fa4e83397fb8d3ee0a6c Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Mon, 13 Nov 2017 00:40:32 +0100 Subject: [PATCH] Subprocess progress Also add an enviornment variable to Script --- jedi/api/__init__.py | 24 +++- jedi/evaluate/__init__.py | 9 +- jedi/evaluate/compiled/subprocess/__init__.py | 120 ++++++++++++++---- jedi/evaluate/project.py | 2 + 4 files changed, 129 insertions(+), 26 deletions(-) diff --git a/jedi/api/__init__.py b/jedi/api/__init__.py index 871dc84b..a98f53a4 100644 --- a/jedi/api/__init__.py +++ b/jedi/api/__init__.py @@ -36,6 +36,7 @@ from jedi.evaluate.syntax_tree import tree_name_to_contexts from jedi.evaluate.context import ModuleContext from jedi.evaluate.context.module import ModuleName from jedi.evaluate.context.iterable import unpack_tuple_to_dict +from jedi.evaluate.compiled.subprocess import get_subprocess # Jedi uses lots and lots of recursion. By setting this a little bit higher, we # can remove some "maximum recursion depth" errors. @@ -79,10 +80,12 @@ class Script(object): :type encoding: str :param sys_path: ``sys.path`` to use during analysis of the script :type sys_path: list + :param environment: TODO + :type sys_path: Environment """ def __init__(self, source=None, line=None, column=None, path=None, - encoding='utf-8', sys_path=None): + encoding='utf-8', sys_path=None, environment=None): self._orig_path = path # An empty path (also empty string) should always result in no path. self.path = os.path.abspath(path) if path else None @@ -112,10 +115,24 @@ class Script(object): # Load the Python grammar of the current interpreter. self._grammar = parso.load_grammar() project = Project(sys_path=sys_path) - self._evaluator = Evaluator(self._grammar, project) + if isinstance(self, Interpreter): + # It's not possible to use a subprocess for the interpreter. + self._compiled_subprocess = None + else: + if environment is None: + executable = sys.executable + else: + executable = environment.executable + self._compiled_subprocess = get_subprocess(executable) + self._evaluator = Evaluator(self._grammar, project, self._compiled_subprocess) project.add_script_path(self.path) debug.speed('init') + def __del__(self): + if self._compiled_subprocess is not None: + self._compiled_subprocess.delete_evaluator(evaluator) + self._evaluator.cleanup_evaluator + @cache.memoize_method def _get_module_node(self): return self._grammar.parse( @@ -356,6 +373,9 @@ class Interpreter(Script): except Exception: raise TypeError("namespaces must be a non-empty list of dicts.") + if 'environment' in kwds: + raise TypeError("Environments are not allowed when using an interpreter.") + super(Interpreter, self).__init__(source, **kwds) self.namespaces = namespaces diff --git a/jedi/evaluate/__init__.py b/jedi/evaluate/__init__.py index d1453f32..ce30e8ae 100644 --- a/jedi/evaluate/__init__.py +++ b/jedi/evaluate/__init__.py @@ -83,10 +83,11 @@ from jedi.evaluate.context import ClassContext, FunctionContext, \ from jedi.evaluate.context.iterable import CompForContext from jedi.evaluate.syntax_tree import eval_trailer, eval_expr_stmt, \ eval_node, check_tuple_assignments +from jedi.evaluate.compiled.subprocess import EvaluatorSubprocess, EvaluatorSameProcess class Evaluator(object): - def __init__(self, grammar, project): + def __init__(self, grammar, project, compiled_sub_process=None): self.grammar = grammar self.latest_grammar = parso.load_grammar(version='3.6') self.memoize_cache = {} # for memoize decorators @@ -102,6 +103,12 @@ class Evaluator(object): self.project = project project.add_evaluator(self) + if compiled_sub_process is None: + self.compiled_subprocess = EvaluatorSameProcess(self) + else: + self.compiled_subprocess = EvaluatorSubprocess(self, compiled_sub_process) + + self.reset_recursion_limitations() # Constants diff --git a/jedi/evaluate/compiled/subprocess/__init__.py b/jedi/evaluate/compiled/subprocess/__init__.py index 64fbb3df..14a1cc6f 100644 --- a/jedi/evaluate/compiled/subprocess/__init__.py +++ b/jedi/evaluate/compiled/subprocess/__init__.py @@ -9,21 +9,69 @@ goals: import sys import subprocess +import weakref import pickle +from functools import partial + +from jedi.cache import memoize_method +from jedi.evaluate.compiled.subprocess import commands _PICKLE_PROTOCOL = 2 +_subprocesses = {} -class _SubProcess(object): + +def get_subprocess(executable): + try: + return _subprocesses[executable] + except KeyError: + sub = _subprocesses[executable] = _CompiledSubprocess(executable) + return sub + + +class EvaluatorSameProcess(object): + """ + Basically just an easy access to functions.py. It has the same API + as EvaluatorSubprocess and does the same thing without using a subprocess. + This is necessary for the Interpreter process. + """ + def __init__(self, evaluator): + self._evaluator = evaluator + + def __getattr__(self): + function = getattr(commands, name) + return partial(function, self._evaluator) + + +class EvaluatorSubprocess(object): + def __init__(self, evaluator, compiled_subprocess): + self._evaluator_weakref = weakref.ref(evaluator) + self._evaluator_id = () + self._compiled_subprocess = compiled_subprocess + + def __getattr__(self, name): + function = getattr(commands, name) + return partial(function, self._evaluator_weakref()) + + def __del__(self): + self.delete_evaluator(self._evaluator_weakref() + + +class _Subprocess(object): def __init__(self, args): - self._process = subprocess.Popen( - args, + self._args = args + + @property + @memoize_method + def _process(self): + return subprocess.Popen( + self._args, stdin=subprocess.PIPE, stdout=subprocess.PIPE, #stderr=subprocess.PIPE ) - def _send(self, data): + def _send(self, *data): pickle.dump(data, self._process.stdin, protocol=_PICKLE_PROTOCOL) self._process.stdin.flush() return pickle.load(self._process.stdout) @@ -35,30 +83,56 @@ class _SubProcess(object): self._process.kill() -class CompiledSubProcess(object): +class _CompiledSubprocess(_Subprocess): def __init__(self, executable): - super(CompiledSubProcess, self).__init__( + super(_CompiledSubprocess, self).__init__( (executable, '-m', 'jedi.evaluate.compiled.subprocess') ) - def command(self, command): - return self._send() + def run(self, evaluator, function, *args, **kwargs): + assert callable(function) + return self._send(id(evaluator), function, args, kwargs) + + def delete_evaluator(self, evaluator_id): + # With an argument - the evaluator gets deleted. + self._send(evaluator_id, None) -def listen(): - stdout = sys.stdout - stdin = sys.stdin - if sys.version_info[0] > 2: - stdout = stdout.buffer - stdin = stdin.buffer +class Listener(): + def __init__(self): + self._evaluators = {} + + def _run(self, evaluator_id, function, args, kwargs): + from jedi.evaluate import Evaluator + + if function is None: + # If the function is None, this is the hint to delete the + # evaluator. + del self._evaluators[evaluator_id] + return - while True: try: - result = pickle.load(stdin) - except EOFError: - # It looks like the parent process closed. Don't make a big fuss - # here and just exit. - exit(1) - result += 1 - pickle.dump(result, stdout, protocol=_PICKLE_PROTOCOL) - stdout.flush() + evaluator = self.evaluators[evaluator_id] + except KeyError: + evaluator = Evaluator(None, None) + self.evaluators[evaluator_id] = evaluator + + return function(evaluator, *args, **kwargs) + + def listen(self): + stdout = sys.stdout + stdin = sys.stdin + if sys.version_info[0] > 2: + stdout = stdout.buffer + stdin = stdin.buffer + + while True: + try: + payload = pickle.load(stdin) + except EOFError: + # It looks like the parent process closed. Don't make a big fuss + # here and just exit. + exit(1) + result = self._run(*payload) + pickle.dump(result, stdout, protocol=_PICKLE_PROTOCOL) + stdout.flush() diff --git a/jedi/evaluate/project.py b/jedi/evaluate/project.py index fe3fb5d7..8a74538c 100644 --- a/jedi/evaluate/project.py +++ b/jedi/evaluate/project.py @@ -3,10 +3,12 @@ import sys from jedi.evaluate.sys_path import get_venv_path, detect_additional_paths from jedi.cache import memoize_method +from jedi.evaluate.compiled.subprocess import CompiledSubProcess class Project(object): def __init__(self, sys_path=None): + self._compiled_subprocess = CompiledSubProcess() if sys_path is not None: self._sys_path = sys_path