Make the subprocesses work and return the right sys paths for the different versions

jedi/_compatibility.py

@@ -289,25 +289,3 @@ def utf8_repr(func):
         return func
     else:
         return wrapper
-
-
-try:
-    from subprocess import check_output
-except ImportError:
-    def check_output(*popenargs, **kwargs):
-        """
-        Taken from Python 2.6.
-        """
-        from subprocess import Popen, PIPE, CalledProcessError
-
-        if 'stdout' in kwargs:
-            raise ValueError('stdout argument not allowed, it will be overridden.')
-        process = Popen(stdout=PIPE, *popenargs, **kwargs)
-        output, unused_err = process.communicate()
-        retcode = process.poll()
-        if retcode:
-            cmd = kwargs.get("args")
-            if cmd is None:
-                cmd = popenargs[0]
-            raise CalledProcessError(retcode, cmd, output=output)
-        return output

jedi/api/virtualenv.py

@@ -1,11 +1,9 @@
 import os
 import re
 import sys
-import sysconfig
-from subprocess import CalledProcessError
+from subprocess import Popen, PIPE
 from collections import namedtuple
 
-from jedi._compatibility import check_output
 from jedi.evaluate.project import Project
 from jedi.cache import memoize_method
 from jedi.evaluate.compiled.subprocess import get_subprocess
@@ -34,16 +32,11 @@ class Environment(object):
 
     @memoize_method
     def get_sys_path(self):
-        vars = {
-            'base': self._path
-        }
-        lib_path = sysconfig.get_path('stdlib', vars=vars)
         # It's pretty much impossible to generate the sys path without actually
         # executing Python. The sys path (when starting with -S) itself depends
         # on how the Python version was compiled (ENV variables).
         # If you omit -S when starting Python (normal case), additionally
         # site.py gets executed.
-
         return self.get_subprocess().get_sys_path()
 
 
@@ -80,11 +73,18 @@ def _get_executable_path(path):
 
 def _get_version(executable):
     try:
-        output = check_output([executable, '--version'])
-    except (CalledProcessError, OSError):
+        process = Popen([executable, '--version'], stdout=PIPE, stderr=PIPE)
+        stdout, stderr = process.communicate()
+        retcode = process.poll()
+        if retcode:
+            raise NoVirtualEnv()
+    except OSError:
         raise NoVirtualEnv()
 
-    match = re.match(rb'Python (\d+)\.(\d+)\.(\d+)', output)
+    # Until Python 3.4 wthe version string is part of stderr, after that
+    # stdout.
+    output = stdout + stderr
+    match = re.match(br'Python (\d+)\.(\d+)\.(\d+)', output)
     if match is None:
         raise NoVirtualEnv()
 

jedi/evaluate/compiled/subprocess/__init__.py

@@ -7,6 +7,7 @@ goals:
 2. Make it possible to handle different Python versions as well as virtualenvs.
 """
 
+import os
 import sys
 import subprocess
 import weakref
@@ -20,6 +21,8 @@ _PICKLE_PROTOCOL = 2
 
 _subprocesses = {}
 
+_MAIN_PATH = os.path.join(os.path.dirname(__file__), '__main__.py')
+
 
 def get_subprocess(executable):
     try:
@@ -88,8 +91,12 @@ class _Subprocess(object):
 
 class _CompiledSubprocess(_Subprocess):
     def __init__(self, executable):
+        parso_path = sys.modules['parso'].__file__
         super(_CompiledSubprocess, self).__init__(
-            (executable, '-m', 'jedi.evaluate.compiled.subprocess')
+            (executable,
+             _MAIN_PATH,
+             os.path.dirname(os.path.dirname(parso_path))
+             )
         )
 
     def run(self, evaluator, function, *args, **kwargs):

jedi/evaluate/compiled/subprocess/__main__.py

@@ -1,3 +1,15 @@
+import sys
+import os
+
+# Get the path to jedi.
+_d = os.path.dirname
+_jedi_path = _d(_d(_d(_d(_d(__file__)))))
+_parso_path = sys.argv[1]
+
+# This is kind of stupid. We actually don't want to modify the sys path but
+# simply import something from a specific location.
+sys.path[0:0] = [_jedi_path, _parso_path]
 from jedi.evaluate.compiled import subprocess
+sys.path[0:2] = []
 
 subprocess.Listener().listen()

jedi/evaluate/site.py

@@ -17,7 +17,6 @@ from __future__ import print_function
 
 import sys
 import os
-from sysconfig import get_config_var
 
 
 def makepath(*paths):
@@ -109,8 +108,3 @@ def addsitedir(sys_path, sitedir, known_paths=None):
     if reset:
         known_paths = None
     return known_paths
-
-
-def getuserbase():
-    """Returns the `user base` directory path."""
-    return get_config_var('userbase')

test/test_api/test_environment.py

@@ -1,5 +1,26 @@
-from jedi.api.virtualenv import Environment, DefaultEnvironment
+import pytest
+
+from jedi._compatibility import py_version
+from jedi.api.virtualenv import Environment, DefaultEnvironment, NoVirtualEnv
 
 
 def test_sys_path():
     assert DefaultEnvironment('/foo').get_sys_path()
+
+
+@pytest.mark.parametrize(
+    'version',
+    ['2.6', '2.7', '3.3', '3.4', '3.5', '3.6', '3.7']
+)
+def test_versions(version):
+    executable = 'python' + version
+    try:
+        env = Environment('some path', executable)
+    except NoVirtualEnv:
+        if int(version.replace('.', '')) == py_version:
+            # At least the current version has to work
+            raise
+        return
+
+    sys_path = env.get_sys_path()
+    assert any(executable in p for p in sys_path)