Use highest possible pickle protocol

jedi/_compatibility.py

@@ -446,9 +446,6 @@ if sys.version_info[:2] == (3, 3):
     pickle.loads = loads
 
 
-_PICKLE_PROTOCOL = 2
-
-
 def pickle_load(file):
     try:
         if is_py3:
@@ -462,9 +459,9 @@ def pickle_load(file):
         raise
 
 
-def pickle_dump(data, file):
+def pickle_dump(data, file, protocol):
     try:
-        pickle.dump(data, file, protocol=_PICKLE_PROTOCOL)
+        pickle.dump(data, file, protocol)
         # On Python 3.3 flush throws sometimes an error even though the writing
         # operation should be completed.
         file.flush()
@@ -476,6 +473,20 @@ def pickle_dump(data, file):
         raise
 
 
+# Determine the highest protocol version compatible for a given list of Python
+# versions.
+def highest_pickle_protocol(python_versions):
+    protocol = 4
+    for version in python_versions:
+        if version[0] == 2:
+            # The minimum protocol version for the versions of Python that we
+            # support (2.7 and 3.3+) is 2.
+            return 2
+        if version[1] < 4:
+            protocol = 3
+    return protocol
+
+
 try:
     from inspect import Parameter
 except ImportError:

jedi/api/environment.py

@@ -98,7 +98,7 @@ class Environment(_BaseEnvironment):
         return EvaluatorSubprocess(evaluator, self._get_subprocess())
 
     def _get_subprocess(self):
-        return get_subprocess(self.executable)
+        return get_subprocess(self.executable, self.version_info)
 
     @memoize_method
     def get_sys_path(self):

jedi/evaluate/compiled/subprocess/__init__.py

@@ -17,7 +17,7 @@ import traceback
 from functools import partial
 
 from jedi._compatibility import queue, is_py3, force_unicode, \
-    pickle_dump, pickle_load, GeneralizedPopen
+    pickle_dump, pickle_load, highest_pickle_protocol, GeneralizedPopen
 from jedi.cache import memoize_method
 from jedi.evaluate.compiled.subprocess import functions
 from jedi.evaluate.compiled.access import DirectObjectAccess, AccessPath, \
@@ -29,11 +29,12 @@ _subprocesses = {}
 _MAIN_PATH = os.path.join(os.path.dirname(__file__), '__main__.py')
 
 
-def get_subprocess(executable):
+def get_subprocess(executable, version):
     try:
         return _subprocesses[executable]
     except KeyError:
-        sub = _subprocesses[executable] = _CompiledSubprocess(executable)
+        sub = _subprocesses[executable] = _CompiledSubprocess(executable,
+                                                              version)
         return sub
 
 
@@ -125,9 +126,11 @@ class EvaluatorSubprocess(_EvaluatorProcess):
 class _CompiledSubprocess(object):
     _crashed = False
 
-    def __init__(self, executable):
+    def __init__(self, executable, version):
         self._executable = executable
         self._evaluator_deletion_queue = queue.deque()
+        self._pickle_protocol = highest_pickle_protocol([sys.version_info,
+                                                         version])
 
     @property
     @memoize_method
@@ -136,7 +139,8 @@ class _CompiledSubprocess(object):
         args = (
             self._executable,
             _MAIN_PATH,
-            os.path.dirname(os.path.dirname(parso_path))
+            os.path.dirname(os.path.dirname(parso_path)),
+            str(self._pickle_protocol)
         )
         return GeneralizedPopen(
             args,
@@ -190,7 +194,7 @@ class _CompiledSubprocess(object):
 
         data = evaluator_id, function, args, kwargs
         try:
-            pickle_dump(data, self._process.stdin)
+            pickle_dump(data, self._process.stdin, self._pickle_protocol)
         except (socket.error, IOError) as e:
             # Once Python2 will be removed we can just use `BrokenPipeError`.
             # Also, somehow in windows it returns EINVAL instead of EPIPE if
@@ -236,11 +240,12 @@ class _CompiledSubprocess(object):
 
 
 class Listener(object):
-    def __init__(self):
+    def __init__(self, pickle_protocol):
         self._evaluators = {}
         # TODO refactor so we don't need to process anymore just handle
         # controlling.
         self._process = _EvaluatorProcess(Listener)
+        self._pickle_protocol = pickle_protocol
 
     def _get_evaluator(self, function, evaluator_id):
         from jedi.evaluate import Evaluator
@@ -307,7 +312,7 @@ class Listener(object):
             except Exception as e:
                 result = True, traceback.format_exc(), e
 
-            pickle_dump(result, file=stdout)
+            pickle_dump(result, stdout, self._pickle_protocol)
 
 
 class AccessHandle(object):

jedi/evaluate/compiled/subprocess/__main__.py

@@ -45,5 +45,7 @@ else:
     load('jedi')
     from jedi.evaluate.compiled import subprocess  # NOQA
 
+# Retrieve the pickle protocol.
+pickle_protocol = int(sys.argv[2])
 # And finally start the client.
-subprocess.Listener().listen()
+subprocess.Listener(pickle_protocol).listen()

test/test_compatibility.py

@@ -0,0 +1,26 @@
+from collections import namedtuple
+from jedi._compatibility import highest_pickle_protocol
+
+
+def test_highest_pickle_protocol():
+    v = namedtuple('version', 'major, minor')
+    assert highest_pickle_protocol([v(2, 7), v(2, 7)]) == 2
+    assert highest_pickle_protocol([v(2, 7), v(3, 3)]) == 2
+    assert highest_pickle_protocol([v(2, 7), v(3, 4)]) == 2
+    assert highest_pickle_protocol([v(2, 7), v(3, 5)]) == 2
+    assert highest_pickle_protocol([v(2, 7), v(3, 6)]) == 2
+    assert highest_pickle_protocol([v(3, 3), v(2, 7)]) == 2
+    assert highest_pickle_protocol([v(3, 3), v(3, 3)]) == 3
+    assert highest_pickle_protocol([v(3, 3), v(3, 4)]) == 3
+    assert highest_pickle_protocol([v(3, 3), v(3, 5)]) == 3
+    assert highest_pickle_protocol([v(3, 3), v(3, 6)]) == 3
+    assert highest_pickle_protocol([v(3, 4), v(2, 7)]) == 2
+    assert highest_pickle_protocol([v(3, 4), v(3, 3)]) == 3
+    assert highest_pickle_protocol([v(3, 4), v(3, 4)]) == 4
+    assert highest_pickle_protocol([v(3, 4), v(3, 5)]) == 4
+    assert highest_pickle_protocol([v(3, 4), v(3, 6)]) == 4
+    assert highest_pickle_protocol([v(3, 6), v(2, 7)]) == 2
+    assert highest_pickle_protocol([v(3, 6), v(3, 3)]) == 3
+    assert highest_pickle_protocol([v(3, 6), v(3, 4)]) == 4
+    assert highest_pickle_protocol([v(3, 6), v(3, 5)]) == 4
+    assert highest_pickle_protocol([v(3, 6), v(3, 6)]) == 4