From 2fc91ceb64aa6adc22a3f21fea65a61dbad54884 Mon Sep 17 00:00:00 2001 From: Daniel Hahler Date: Fri, 27 Apr 2018 10:05:46 +0200 Subject: [PATCH] Improve Environment It only takes `executable` and gets all the information from the subprocess directly. Fixes https://github.com/davidhalter/jedi/issues/1107. --- jedi/api/environment.py | 109 ++++++++---------- jedi/evaluate/compiled/subprocess/__init__.py | 24 ++-- jedi/evaluate/compiled/subprocess/__main__.py | 10 +- 3 files changed, 69 insertions(+), 74 deletions(-) diff --git a/jedi/api/environment.py b/jedi/api/environment.py index 57397f37..06eb90ec 100644 --- a/jedi/api/environment.py +++ b/jedi/api/environment.py @@ -3,14 +3,12 @@ Environments are a way to activate different Python versions or Virtualenvs for static analysis. The Python binary in that environment is going to be executed. """ import os -import re import sys import hashlib import filecmp -from subprocess import PIPE from collections import namedtuple -from jedi._compatibility import GeneralizedPopen, which +from jedi._compatibility import highest_pickle_protocol, which from jedi.cache import memoize_method, time_cache from jedi.evaluate.compiled.subprocess import get_subprocess, \ EvaluatorSameProcess, EvaluatorSubprocess @@ -46,49 +44,52 @@ class _BaseEnvironment(object): return self._hash +def _get_info(): + return ( + sys.executable, + sys.prefix, + sys.version_info[:3], + ) + + class Environment(_BaseEnvironment): """ This class is supposed to be created by internal Jedi architecture. You should not create it directly. Please use create_environment or the other functions instead. It is then returned by that function. """ - def __init__(self, path, executable): - self.path = os.path.abspath(path) - """ - The path to an environment, matches ``sys.prefix``. - """ - self.executable = os.path.abspath(executable) + def __init__(self, executable): + try: + self._subprocess = get_subprocess(executable) + info = self._subprocess._send(None, _get_info) + except Exception as exc: + raise InvalidPythonEnvironment( + "Could not get version information for %r: %r" % ( + executable, + exc)) + + self.executable = info[0] """ The Python executable, matches ``sys.executable``. """ - self.version_info = self._get_version() + self.path = info[1] + """ + The path to an environment, matches ``sys.prefix``. + """ + self.version_info = _VersionInfo(*info[2]) """ - Like ``sys.version_info``. A tuple to show the current Environment's Python version. """ - def _get_version(self): - try: - process = GeneralizedPopen([self.executable, '--version'], stdout=PIPE, stderr=PIPE) - stdout, stderr = process.communicate() - retcode = process.poll() - if retcode: - raise InvalidPythonEnvironment( - "Exited with %d (stdout=%r, stderr=%r)" % ( - retcode, stdout, stderr)) - except OSError as exc: - raise InvalidPythonEnvironment( - "Could not get version information: %r" % exc) + # Adjust pickle protocol according to host and client version. + self._subprocess._pickle_protocol = highest_pickle_protocol([ + sys.version_info, self.version_info]) - # Until Python 3.4 wthe version string is part of stderr, after that - # stdout. - output = stdout + stderr - match = re.match(br'Python (\d+)\.(\d+)\.(\d+)', output) - if match is None: - raise InvalidPythonEnvironment("--version not working") - - return _VersionInfo(*[int(m) for m in match.groups()]) + # py2 sends bytes via pickle apparently?! + if self.version_info.major == 2: + self.executable = self.executable.decode() + self.path = self.path.decode() def __repr__(self): version = '.'.join(str(i) for i in self.version_info) @@ -98,7 +99,7 @@ class Environment(_BaseEnvironment): return EvaluatorSubprocess(evaluator, self._get_subprocess()) def _get_subprocess(self): - return get_subprocess(self.executable, self.version_info) + return get_subprocess(self.executable) @memoize_method def get_sys_path(self): @@ -118,10 +119,9 @@ class Environment(_BaseEnvironment): class SameEnvironment(Environment): def __init__(self): - super(SameEnvironment, self).__init__(sys.prefix, sys.executable) - - def _get_version(self): - return _VersionInfo(*sys.version_info[:3]) + self.executable = sys.executable + self.path = sys.prefix + self.version_info = _VersionInfo(*sys.version_info[:3]) class InterpreterEnvironment(_BaseEnvironment): @@ -222,7 +222,7 @@ def find_virtualenvs(paths=None, **kwargs): try: executable = _get_executable_path(path, safe=safe) - yield Environment(path, executable) + yield Environment(executable) except InvalidPythonEnvironment: pass @@ -246,23 +246,6 @@ def find_system_environments(): pass -# TODO: the logic to find the Python prefix is much more complicated than that. -# See Modules/getpath.c for UNIX and PC/getpathp.c for Windows in CPython's -# source code. A solution would be to deduce it by running the Python -# interpreter and printing the value of sys.prefix. -def _get_python_prefix(executable): - if os.name != 'nt': - return os.path.dirname(os.path.dirname(executable)) - landmark = os.path.join('Lib', 'os.py') - prefix = os.path.dirname(executable) - while prefix: - if os.path.join(prefix, landmark): - return prefix - prefix = os.path.dirname(prefix) - raise InvalidPythonEnvironment( - "Cannot find prefix of executable %s." % executable) - - # TODO: this function should probably return a list of environments since # multiple Python installations can be found on a system for the same version. def get_system_environment(version): @@ -277,17 +260,17 @@ def get_system_environment(version): if exe: if exe == sys.executable: return SameEnvironment() - return Environment(_get_python_prefix(exe), exe) + return Environment(exe) if os.name == 'nt': - for prefix, exe in _get_executables_from_windows_registry(version): - return Environment(prefix, exe) + for exe in _get_executables_from_windows_registry(version): + return Environment(exe) raise InvalidPythonEnvironment("Cannot find executable python%s." % version) def create_environment(path, safe=True): """ - Make it possible to manually create an environment by specifying a + Make it possible to manually create an Environment object by specifying a Virtualenv path or an executable path. :raises: :exc:`.InvalidPythonEnvironment` @@ -295,8 +278,8 @@ def create_environment(path, safe=True): """ if os.path.isfile(path): _assert_safe(path, safe) - return Environment(_get_python_prefix(path), path) - return Environment(path, _get_executable_path(path, safe=safe)) + return Environment(path) + return Environment(_get_executable_path(path, safe=safe)) def _get_executable_path(path, safe=True): @@ -318,9 +301,9 @@ def _get_executable_path(path, safe=True): def _get_executables_from_windows_registry(version): # The winreg module is named _winreg on Python 2. try: - import winreg + import winreg except ImportError: - import _winreg as winreg + import _winreg as winreg # TODO: support Python Anaconda. sub_keys = [ @@ -337,7 +320,7 @@ def _get_executables_from_windows_registry(version): prefix = winreg.QueryValueEx(key, '')[0] exe = os.path.join(prefix, 'python.exe') if os.path.isfile(exe): - yield prefix, exe + yield exe except WindowsError: pass diff --git a/jedi/evaluate/compiled/subprocess/__init__.py b/jedi/evaluate/compiled/subprocess/__init__.py index 3fde38d3..71edb77b 100644 --- a/jedi/evaluate/compiled/subprocess/__init__.py +++ b/jedi/evaluate/compiled/subprocess/__init__.py @@ -17,7 +17,7 @@ import traceback from functools import partial from jedi._compatibility import queue, is_py3, force_unicode, \ - pickle_dump, pickle_load, highest_pickle_protocol, GeneralizedPopen + pickle_dump, pickle_load, GeneralizedPopen from jedi.cache import memoize_method from jedi.evaluate.compiled.subprocess import functions from jedi.evaluate.compiled.access import DirectObjectAccess, AccessPath, \ @@ -29,12 +29,11 @@ _subprocesses = {} _MAIN_PATH = os.path.join(os.path.dirname(__file__), '__main__.py') -def get_subprocess(executable, version): +def get_subprocess(executable): try: return _subprocesses[executable] except KeyError: - sub = _subprocesses[executable] = _CompiledSubprocess(executable, - version) + sub = _subprocesses[executable] = _CompiledSubprocess(executable) return sub @@ -125,12 +124,21 @@ class EvaluatorSubprocess(_EvaluatorProcess): class _CompiledSubprocess(object): _crashed = False + # Start with 2, gets set after _get_info. + _pickle_protocol = 2 - def __init__(self, executable, version): + def __init__(self, executable): self._executable = executable self._evaluator_deletion_queue = queue.deque() - self._pickle_protocol = highest_pickle_protocol([sys.version_info, - version]) + + def __repr__(self): + pid = os.getpid() + return '<_CompiledSubprocess _executable=%r, _pickle_protocol=%r, _crashed=%r, pid=%r>' % ( + self._executable, + self._pickle_protocol, + self._crashed, + pid, + ) @property @memoize_method @@ -140,7 +148,7 @@ class _CompiledSubprocess(object): self._executable, _MAIN_PATH, os.path.dirname(os.path.dirname(parso_path)), - str(self._pickle_protocol) + '.'.join(str(x) for x in sys.version_info[:3]), ) return GeneralizedPopen( args, diff --git a/jedi/evaluate/compiled/subprocess/__main__.py b/jedi/evaluate/compiled/subprocess/__main__.py index ff7462fa..4be28204 100644 --- a/jedi/evaluate/compiled/subprocess/__main__.py +++ b/jedi/evaluate/compiled/subprocess/__main__.py @@ -1,5 +1,5 @@ -import sys import os +import sys def _get_paths(): @@ -45,7 +45,11 @@ else: load('jedi') from jedi.evaluate.compiled import subprocess # NOQA +from jedi._compatibility import highest_pickle_protocol # noqa: E402 + + # Retrieve the pickle protocol. -pickle_protocol = int(sys.argv[2]) +host_sys_version = [int(x) for x in sys.argv[2].split('.')] +pickle_protocol = highest_pickle_protocol([sys.version_info, host_sys_version]) # And finally start the client. -subprocess.Listener(pickle_protocol).listen() +subprocess.Listener(pickle_protocol=pickle_protocol).listen()