From 282c6a2ba1d5bf175342a8a09b6428071a7e3856 Mon Sep 17 00:00:00 2001 From: micbou Date: Wed, 20 Jun 2018 14:55:04 +0200 Subject: [PATCH] Use highest possible pickle protocol --- jedi/_compatibility.py | 21 +++++++++++---- jedi/api/environment.py | 2 +- jedi/evaluate/compiled/subprocess/__init__.py | 21 +++++++++------ jedi/evaluate/compiled/subprocess/__main__.py | 4 ++- test/test_compatibility.py | 26 +++++++++++++++++++ 5 files changed, 59 insertions(+), 15 deletions(-) create mode 100644 test/test_compatibility.py diff --git a/jedi/_compatibility.py b/jedi/_compatibility.py index d8494a25..2002a1ca 100644 --- a/jedi/_compatibility.py +++ b/jedi/_compatibility.py @@ -446,9 +446,6 @@ if sys.version_info[:2] == (3, 3): pickle.loads = loads -_PICKLE_PROTOCOL = 2 - - def pickle_load(file): try: if is_py3: @@ -462,9 +459,9 @@ def pickle_load(file): raise -def pickle_dump(data, file): +def pickle_dump(data, file, protocol): try: - pickle.dump(data, file, protocol=_PICKLE_PROTOCOL) + pickle.dump(data, file, protocol) # On Python 3.3 flush throws sometimes an error even though the writing # operation should be completed. file.flush() @@ -476,6 +473,20 @@ def pickle_dump(data, file): raise +# Determine the highest protocol version compatible for a given list of Python +# versions. +def highest_pickle_protocol(python_versions): + protocol = 4 + for version in python_versions: + if version[0] == 2: + # The minimum protocol version for the versions of Python that we + # support (2.7 and 3.3+) is 2. + return 2 + if version[1] < 4: + protocol = 3 + return protocol + + try: from inspect import Parameter except ImportError: diff --git a/jedi/api/environment.py b/jedi/api/environment.py index e866b888..57397f37 100644 --- a/jedi/api/environment.py +++ b/jedi/api/environment.py @@ -98,7 +98,7 @@ class Environment(_BaseEnvironment): return EvaluatorSubprocess(evaluator, self._get_subprocess()) def _get_subprocess(self): - return get_subprocess(self.executable) + return get_subprocess(self.executable, self.version_info) @memoize_method def get_sys_path(self): diff --git a/jedi/evaluate/compiled/subprocess/__init__.py b/jedi/evaluate/compiled/subprocess/__init__.py index 0f8c328c..e088560d 100644 --- a/jedi/evaluate/compiled/subprocess/__init__.py +++ b/jedi/evaluate/compiled/subprocess/__init__.py @@ -17,7 +17,7 @@ import traceback from functools import partial from jedi._compatibility import queue, is_py3, force_unicode, \ - pickle_dump, pickle_load, GeneralizedPopen + pickle_dump, pickle_load, highest_pickle_protocol, GeneralizedPopen from jedi.cache import memoize_method from jedi.evaluate.compiled.subprocess import functions from jedi.evaluate.compiled.access import DirectObjectAccess, AccessPath, \ @@ -29,11 +29,12 @@ _subprocesses = {} _MAIN_PATH = os.path.join(os.path.dirname(__file__), '__main__.py') -def get_subprocess(executable): +def get_subprocess(executable, version): try: return _subprocesses[executable] except KeyError: - sub = _subprocesses[executable] = _CompiledSubprocess(executable) + sub = _subprocesses[executable] = _CompiledSubprocess(executable, + version) return sub @@ -125,9 +126,11 @@ class EvaluatorSubprocess(_EvaluatorProcess): class _CompiledSubprocess(object): _crashed = False - def __init__(self, executable): + def __init__(self, executable, version): self._executable = executable self._evaluator_deletion_queue = queue.deque() + self._pickle_protocol = highest_pickle_protocol([sys.version_info, + version]) @property @memoize_method @@ -136,7 +139,8 @@ class _CompiledSubprocess(object): args = ( self._executable, _MAIN_PATH, - os.path.dirname(os.path.dirname(parso_path)) + os.path.dirname(os.path.dirname(parso_path)), + str(self._pickle_protocol) ) return GeneralizedPopen( args, @@ -190,7 +194,7 @@ class _CompiledSubprocess(object): data = evaluator_id, function, args, kwargs try: - pickle_dump(data, self._process.stdin) + pickle_dump(data, self._process.stdin, self._pickle_protocol) except (socket.error, IOError) as e: # Once Python2 will be removed we can just use `BrokenPipeError`. # Also, somehow in windows it returns EINVAL instead of EPIPE if @@ -236,11 +240,12 @@ class _CompiledSubprocess(object): class Listener(object): - def __init__(self): + def __init__(self, pickle_protocol): self._evaluators = {} # TODO refactor so we don't need to process anymore just handle # controlling. self._process = _EvaluatorProcess(Listener) + self._pickle_protocol = pickle_protocol def _get_evaluator(self, function, evaluator_id): from jedi.evaluate import Evaluator @@ -307,7 +312,7 @@ class Listener(object): except Exception as e: result = True, traceback.format_exc(), e - pickle_dump(result, file=stdout) + pickle_dump(result, stdout, self._pickle_protocol) class AccessHandle(object): diff --git a/jedi/evaluate/compiled/subprocess/__main__.py b/jedi/evaluate/compiled/subprocess/__main__.py index 8a0148ce..ff7462fa 100644 --- a/jedi/evaluate/compiled/subprocess/__main__.py +++ b/jedi/evaluate/compiled/subprocess/__main__.py @@ -45,5 +45,7 @@ else: load('jedi') from jedi.evaluate.compiled import subprocess # NOQA +# Retrieve the pickle protocol. +pickle_protocol = int(sys.argv[2]) # And finally start the client. -subprocess.Listener().listen() +subprocess.Listener(pickle_protocol).listen() diff --git a/test/test_compatibility.py b/test/test_compatibility.py new file mode 100644 index 00000000..01c9dae5 --- /dev/null +++ b/test/test_compatibility.py @@ -0,0 +1,26 @@ +from collections import namedtuple +from jedi._compatibility import highest_pickle_protocol + + +def test_highest_pickle_protocol(): + v = namedtuple('version', 'major, minor') + assert highest_pickle_protocol([v(2, 7), v(2, 7)]) == 2 + assert highest_pickle_protocol([v(2, 7), v(3, 3)]) == 2 + assert highest_pickle_protocol([v(2, 7), v(3, 4)]) == 2 + assert highest_pickle_protocol([v(2, 7), v(3, 5)]) == 2 + assert highest_pickle_protocol([v(2, 7), v(3, 6)]) == 2 + assert highest_pickle_protocol([v(3, 3), v(2, 7)]) == 2 + assert highest_pickle_protocol([v(3, 3), v(3, 3)]) == 3 + assert highest_pickle_protocol([v(3, 3), v(3, 4)]) == 3 + assert highest_pickle_protocol([v(3, 3), v(3, 5)]) == 3 + assert highest_pickle_protocol([v(3, 3), v(3, 6)]) == 3 + assert highest_pickle_protocol([v(3, 4), v(2, 7)]) == 2 + assert highest_pickle_protocol([v(3, 4), v(3, 3)]) == 3 + assert highest_pickle_protocol([v(3, 4), v(3, 4)]) == 4 + assert highest_pickle_protocol([v(3, 4), v(3, 5)]) == 4 + assert highest_pickle_protocol([v(3, 4), v(3, 6)]) == 4 + assert highest_pickle_protocol([v(3, 6), v(2, 7)]) == 2 + assert highest_pickle_protocol([v(3, 6), v(3, 3)]) == 3 + assert highest_pickle_protocol([v(3, 6), v(3, 4)]) == 4 + assert highest_pickle_protocol([v(3, 6), v(3, 5)]) == 4 + assert highest_pickle_protocol([v(3, 6), v(3, 6)]) == 4