Improve virtualenv support & egg-link resolution

- add sys_path= kwarg to Script & Evaluator constructors

- store sys_path for each evaluator instance

- replace get_sys_path with get_venv_path

- get_venv_path: use addsitedir to load .pth extension files

- get_venv_path: look for egg-link files in all directories in path

jedi/api/__init__.py

@@ -34,6 +34,7 @@ from jedi.evaluate.cache import memoize_default
 from jedi.evaluate.helpers import FakeName, get_module_names
 from jedi.evaluate.finder import global_names_dict_generator, filter_definition_names
 from jedi.evaluate import analysis
+from jedi.evaluate.sys_path import get_venv_path
 
 # Jedi uses lots and lots of recursion. By setting this a little bit higher, we
 # can remove some "maximum recursion depth" errors.
@@ -75,7 +76,8 @@ class Script(object):
     :type encoding: str
     """
     def __init__(self, source=None, line=None, column=None, path=None,
-                 encoding='utf-8', source_path=None, source_encoding=None):
+                 encoding='utf-8', source_path=None, source_encoding=None,
+                 sys_path=None):
         if source_path is not None:
             warnings.warn("Use path instead of source_path.", DeprecationWarning)
             path = source_path
@@ -109,7 +111,11 @@ class Script(object):
         self._parser = UserContextParser(self._grammar, self.source, path,
                                          self._pos, self._user_context,
                                          self._parsed_callback)
-        self._evaluator = Evaluator(self._grammar)
+        if sys_path is None:
+            venv = os.getenv('VIRTUAL_ENV')
+            if venv:
+                sys_path = list(get_venv_path(venv))
+        self._evaluator = Evaluator(self._grammar, sys_path=sys_path)
         debug.speed('init')
 
     def _parsed_callback(self, parser):

jedi/evaluate/__init__.py

@@ -61,6 +61,7 @@ that are not used are just being ignored.
 """
 
 import copy
+import sys
 from itertools import chain
 
 from jedi.parser import tree
@@ -79,7 +80,7 @@ from jedi.evaluate import helpers
 
 
 class Evaluator(object):
-    def __init__(self, grammar):
+    def __init__(self, grammar, sys_path=None):
         self.grammar = grammar
         self.memoize_cache = {}  # for memoize decorators
         # To memorize modules -> equals `sys.modules`.
@@ -88,6 +89,13 @@ class Evaluator(object):
         self.recursion_detector = recursion.RecursionDetector()
         self.execution_recursion_detector = recursion.ExecutionRecursionDetector()
         self.analysis = []
+        if sys_path is None:
+            sys_path = sys.path
+        self.sys_path = copy.copy(sys_path)
+        try:
+            self.sys_path.remove('')
+        except ValueError:
+            pass
 
     def wrap(self, element):
         if isinstance(element, tree.Class):

jedi/evaluate/compiled/__init__.py

@@ -10,7 +10,6 @@ from functools import partial
 from jedi._compatibility import builtins as _builtins, unicode
 from jedi import debug
 from jedi.cache import underscore_memoization, memoize_method
-from jedi.evaluate.sys_path import get_sys_path
 from jedi.parser.tree import Param, Base, Operator, zero_position_modifier
 from jedi.evaluate.helpers import FakeName
 from . import fake
@@ -309,15 +308,12 @@ class CompiledName(FakeName):
         pass  # Just ignore this, FakeName tries to overwrite the parent attribute.
 
 
-def dotted_from_fs_path(fs_path, sys_path=None):
+def dotted_from_fs_path(fs_path, sys_path):
     """
     Changes `/usr/lib/python3.4/email/utils.py` to `email.utils`.  I.e.
     compares the path with sys.path and then returns the dotted_path. If the
     path is not in the sys.path, just returns None.
     """
-    if sys_path is None:
-        sys_path = get_sys_path()
-
     if os.path.basename(fs_path).startswith('__init__.'):
         # We are calculating the path. __init__ files are not interesting.
         fs_path = os.path.dirname(fs_path)
@@ -341,13 +337,13 @@ def dotted_from_fs_path(fs_path, sys_path=None):
     return _path_re.sub('', fs_path[len(path):].lstrip(os.path.sep)).replace(os.path.sep, '.')
 
 
-def load_module(path=None, name=None):
+def load_module(evaluator, path=None, name=None):
+    sys_path = evaluator.sys_path
     if path is not None:
-        dotted_path = dotted_from_fs_path(path)
+        dotted_path = dotted_from_fs_path(path, sys_path=sys_path)
     else:
         dotted_path = name
 
-    sys_path = get_sys_path()
     if dotted_path is None:
         p, _, dotted_path = path.partition(os.path.sep)
         sys_path.insert(0, p)

jedi/evaluate/imports.py

@@ -342,7 +342,7 @@ class Importer(object):
             module_file.close()
 
         if module_file is None and not module_path.endswith('.py'):
-            module = compiled.load_module(module_path)
+            module = compiled.load_module(self._evaluator, module_path)
         else:
             module = _load_module(self._evaluator, module_path, source, sys_path)
 
@@ -440,12 +440,15 @@ def _load_module(evaluator, path=None, source=None, sys_path=None):
                 with open(path, 'rb') as f:
                     source = f.read()
         else:
-            return compiled.load_module(path)
+            return compiled.load_module(evaluator, path)
         p = path
         p = fast.FastParser(evaluator.grammar, common.source_to_unicode(source), p)
         cache.save_parser(path, p)
         return p.module
 
+    if sys_path is None:
+        sys_path = evaluator.sys_path
+
     cached = cache.load_parser(path)
     module = load(source) if cached is None else cached.module
     module = evaluator.wrap(module)

jedi/evaluate/sys_path.py

@@ -1,6 +1,7 @@
 import glob
 import os
 import sys
+from site import addsitedir
 
 from jedi._compatibility import exec_function, unicode
 from jedi.parser import tree
@@ -11,24 +12,51 @@ from jedi import common
 from jedi import cache
 
 
-def get_sys_path():
+def get_venv_path(venv):
-    def check_virtual_env(sys_path):
+    """Get sys.path for specified virtual environment."""
-        """ Add virtualenv's site-packages to the `sys.path`."""
+    sys_path = _get_venv_path_dirs(venv)
-        venv = os.getenv('VIRTUAL_ENV')
+    with common.ignored(ValueError):
-        if not venv:
+        sys_path.remove('')
-            return
+    sys_path = _get_sys_path_with_egglinks(sys_path)
-        venv = os.path.abspath(venv)
+    # As of now, get_venv_path_dirs does not scan built-in pythonpath and
-        p = _get_venv_sitepackages(venv)
+    # user-local site-packages, let's approximate them using path from Jedi
-        if p not in sys_path:
+    # interpreter.
-            sys_path.insert(0, p)
+    return sys_path + sys.path
 
-        # Add all egg-links from the virtualenv.
+
+def _get_sys_path_with_egglinks(sys_path):
+    """Find all paths including those referenced by egg-links.
+
+    Egg-link-referenced directories are inserted into path immediately after
+    the directory on which their links were found.  Such directories are not
+    taken into consideration by normal import mechanism, but they are traversed
+    when doing pkg_resources.require.
+    """
+    result = []
+    for p in sys_path:
+        result.append(p)
         for egg_link in glob.glob(os.path.join(p, '*.egg-link')):
             with open(egg_link) as fd:
-                sys_path.insert(0, fd.readline().rstrip())
+                for line in fd:
+                    line = line.strip()
+                    if line:
+                        result.append(os.path.join(p, line))
+                        # pkg_resources package only interprets the first
+                        # non-empty line in egg-link files.
+                        break
+    return result
 
-    check_virtual_env(sys.path)
+
-    return [p for p in sys.path if p != ""]
+def _get_venv_path_dirs(venv):
+    """Get sys.path for venv without starting up the interpreter."""
+    venv = os.path.abspath(venv)
+    sitedir = _get_venv_sitepackages(venv)
+    sys.path, old_sys_path = [], sys.path
+    try:
+        addsitedir(sitedir)
+        return sys.path
+    finally:
+        sys.path = old_sys_path
 
 
 def _get_venv_sitepackages(venv):
@@ -109,7 +137,6 @@ def _paths_from_list_modifications(module_path, trailer1, trailer2):
     name = trailer1.children[1].value
     if name not in ['insert', 'append']:
         return []
-
     arg = trailer2.children[1]
     if name == 'insert' and len(arg.children) in (3, 4):  # Possible trailing comma.
         arg = arg.children[2]
@@ -117,6 +144,9 @@ def _paths_from_list_modifications(module_path, trailer1, trailer2):
 
 
 def _check_module(evaluator, module):
+    """
+    Detect sys.path modifications within module.
+    """
     def get_sys_path_powers(names):
         for name in names:
             power = name.parent.parent
@@ -128,10 +158,12 @@ def _check_module(evaluator, module):
                     if isinstance(n, tree.Name) and n.value == 'path':
                         yield name, power
 
-    sys_path = list(get_sys_path())  # copy
+    sys_path = list(evaluator.sys_path)  # copy
     try:
         possible_names = module.used_names['path']
     except KeyError:
+        # module.used_names is MergedNamesDict whose getitem never throws
+        # keyerror, this is superfluous.
         pass
     else:
         for name, power in get_sys_path_powers(possible_names):
@@ -148,7 +180,7 @@ def sys_path_with_modifications(evaluator, module):
     if module.path is None:
         # Support for modules without a path is bad, therefore return the
         # normal path.
-        return list(get_sys_path())
+        return list(evaluator.sys_path)
 
     curdir = os.path.abspath(os.curdir)
     with common.ignored(OSError):

test/test_evaluate/test_namespace_package.py

@@ -1,55 +1,53 @@
 import jedi
-import sys
 from os.path import dirname, join
 
 
 def test_namespace_package():
-    sys.path.insert(0, join(dirname(__file__), 'namespace_package/ns1'))
+    sys_path = [join(dirname(__file__), d)
-    sys.path.insert(1, join(dirname(__file__), 'namespace_package/ns2'))
+                for d in ['namespace_package/ns1', 'namespace_package/ns2']]
-    try:
-        # goto definition
-        assert jedi.Script('from pkg import ns1_file').goto_definitions()
-        assert jedi.Script('from pkg import ns2_file').goto_definitions()
-        assert not jedi.Script('from pkg import ns3_file').goto_definitions()
 
-        # goto assignment
+    def script_with_path(*args, **kwargs):
-        tests = {
+        return jedi.Script(sys_path=sys_path, *args, **kwargs)
-            'from pkg.ns2_folder.nested import foo': 'nested!',
-            'from pkg.ns2_folder import foo': 'ns2_folder!',
-            'from pkg.ns2_file import foo': 'ns2_file!',
-            'from pkg.ns1_folder import foo': 'ns1_folder!',
-            'from pkg.ns1_file import foo': 'ns1_file!',
-            'from pkg import foo': 'ns1!',
-        }
-        for source, solution in tests.items():
-            ass = jedi.Script(source).goto_assignments()
-            assert len(ass) == 1
-            assert ass[0].description == "foo = '%s'" % solution
 
-        # completion
+    # goto definition
-        completions = jedi.Script('from pkg import ').completions()
+    assert script_with_path('from pkg import ns1_file').goto_definitions()
-        names = [str(c.name) for c in completions]  # str because of unicode
+    assert script_with_path('from pkg import ns2_file').goto_definitions()
-        compare = ['foo', 'ns1_file', 'ns1_folder', 'ns2_folder', 'ns2_file',
+    assert not script_with_path('from pkg import ns3_file').goto_definitions()
-                   'pkg_resources', 'pkgutil', '__name__', '__path__',
-                   '__package__', '__file__', '__doc__']
-        # must at least contain these items, other items are not important
-        assert set(compare) == set(names)
 
-        tests = {
+    # goto assignment
-            'from pkg import ns2_folder as x': 'ns2_folder!',
+    tests = {
-            'from pkg import ns2_file as x': 'ns2_file!',
+        'from pkg.ns2_folder.nested import foo': 'nested!',
-            'from pkg.ns2_folder import nested as x': 'nested!',
+        'from pkg.ns2_folder import foo': 'ns2_folder!',
-            'from pkg import ns1_folder as x': 'ns1_folder!',
+        'from pkg.ns2_file import foo': 'ns2_file!',
-            'from pkg import ns1_file as x': 'ns1_file!',
+        'from pkg.ns1_folder import foo': 'ns1_folder!',
-            'import pkg as x': 'ns1!',
+        'from pkg.ns1_file import foo': 'ns1_file!',
-        }
+        'from pkg import foo': 'ns1!',
-        for source, solution in tests.items():
+    }
-            for c in jedi.Script(source + '; x.').completions():
+    for source, solution in tests.items():
-                if c.name == 'foo':
+        ass = script_with_path(source).goto_assignments()
-                    completion = c
+        assert len(ass) == 1
-            solution = "statement: foo = '%s'" % solution
+        assert ass[0].description == "foo = '%s'" % solution
-            assert completion.description == solution
 
-    finally:
+    # completion
-        sys.path.pop(0)
+    completions = script_with_path('from pkg import ').completions()
-        sys.path.pop(0)
+    names = [str(c.name) for c in completions]  # str because of unicode
+    compare = ['foo', 'ns1_file', 'ns1_folder', 'ns2_folder', 'ns2_file',
+               'pkg_resources', 'pkgutil', '__name__', '__path__',
+               '__package__', '__file__', '__doc__']
+    # must at least contain these items, other items are not important
+    assert set(compare) == set(names)
+
+    tests = {
+        'from pkg import ns2_folder as x': 'ns2_folder!',
+        'from pkg import ns2_file as x': 'ns2_file!',
+        'from pkg.ns2_folder import nested as x': 'nested!',
+        'from pkg import ns1_folder as x': 'ns1_folder!',
+        'from pkg import ns1_file as x': 'ns1_file!',
+        'import pkg as x': 'ns1!',
+    }
+    for source, solution in tests.items():
+        for c in script_with_path(source + '; x.').completions():
+            if c.name == 'foo':
+                completion = c
+        solution = "statement: foo = '%s'" % solution
+        assert completion.description == solution

test/test_evaluate/test_sys_path.py

@@ -1,4 +1,8 @@
 import os
+from glob import glob
+import sys
+
+import pytest
 
 from jedi._compatibility import unicode
 from jedi.parser import Parser, load_grammar
@@ -19,13 +23,30 @@ def test_paths_from_assignment():
     assert paths('sys.path, other = ["a"], 2') == []
 
 
-def test_get_sys_path(monkeypatch):
+# Currently venv site-packages resolution only seeks pythonX.Y/site-packages
-    monkeypatch.setenv('VIRTUAL_ENV', os.path.join(os.path.dirname(__file__),
+# that belong to the same version as the interpreter to avoid issues with
-                                                   'egg-link', 'venv'))
+# cross-version imports.  "venvs/" dir contains "venv27" and "venv34" that
-    def sitepackages_dir(venv):
+# mimic venvs created for py2.7 and py3.4 respectively.  If test runner is
-        return os.path.join(venv, 'lib', 'python3.4', 'site-packages')
+# invoked with one of those versions, the test below will be run for the
+# matching directory.
+CUR_DIR = os.path.dirname(__file__)
+VENVS = list(glob(os.path.join(CUR_DIR,
+                               'venvs/venv%d%d' % sys.version_info[:2])))
 
-    monkeypatch.setattr('jedi.evaluate.sys_path._get_venv_sitepackages',
-                        sitepackages_dir)
 
-    assert '/path/from/egg-link' in sys_path.get_sys_path()
+@pytest.mark.parametrize('venv', VENVS)
+def test_get_venv_path(venv):
+    pjoin = os.path.join
+    venv_path = sys_path.get_venv_path(venv)
+
+    site_pkgs = (glob(pjoin(venv, 'lib', 'python*', 'site-packages')) +
+                 glob(pjoin(venv, 'lib', 'site-packages')))[0]
+    ETALON = [
+        site_pkgs,
+        pjoin(site_pkgs, '.', 'relative', 'egg-link', 'path'),
+        pjoin('/path', 'from', 'egg-link'),
+        pjoin(site_pkgs, 'dir-from-foo-pth'),
+        pjoin('/path', 'from', 'smth.py'),
+        pjoin('/path', 'from', 'smth.py:extend_path')
+    ]
+    assert venv_path[:len(ETALON)] == ETALON

test/test_evaluate/venvs/venv27/lib/python2.7/site-packages/dir-from-foo-pth/__init__.py

@@ -0,0 +1,2 @@
+# This file is here to force git to create the directory, as *.pth files only
+# add existing directories.

test/test_evaluate/venvs/venv27/lib/python2.7/site-packages/foo.pth

@@ -0,0 +1 @@
+./dir-from-foo-pth

test/test_evaluate/venvs/venv27/lib/python2.7/site-packages/import_smth.pth

@@ -0,0 +1 @@
+import smth; smth.extend_path()

test/test_evaluate/venvs/venv27/lib/python2.7/site-packages/relative.egg-link

@@ -0,0 +1 @@
+./relative/egg-link/path

test/test_evaluate/venvs/venv27/lib/python2.7/site-packages/smth.py

@@ -0,0 +1,6 @@
+import sys
+sys.path.append('/path/from/smth.py')
+
+
+def extend_path():
+    sys.path.append('/path/from/smth.py:extend_path')

test/test_evaluate/venvs/venv34/lib/python3.4/site-packages/dir-from-foo-pth/__init__.py

@@ -0,0 +1,2 @@
+# This file is here to force git to create the directory, as *.pth files only
+# add existing directories.

test/test_evaluate/venvs/venv34/lib/python3.4/site-packages/egg_link.egg-link

@@ -0,0 +1 @@
+/path/from/egg-link

test/test_evaluate/venvs/venv34/lib/python3.4/site-packages/foo.pth

@@ -0,0 +1 @@
+./dir-from-foo-pth

test/test_evaluate/venvs/venv34/lib/python3.4/site-packages/import_smth.pth

@@ -0,0 +1 @@
+import smth; smth.extend_path()

test/test_evaluate/venvs/venv34/lib/python3.4/site-packages/relative.egg-link

@@ -0,0 +1 @@
+./relative/egg-link/path

test/test_evaluate/venvs/venv34/lib/python3.4/site-packages/smth.py

@@ -0,0 +1,6 @@
+import sys
+sys.path.append('/path/from/smth.py')
+
+
+def extend_path():
+    sys.path.append('/path/from/smth.py:extend_path')

test/test_regression.py

@@ -66,8 +66,9 @@ class TestRegression(TestCase):
         src1 = "def r(a): return a"
         # Other fictional modules in another place in the fs.
         src2 = 'from .. import setup; setup.r(1)'
-        imports.load_module(os.path.abspath(fname), src2)
+        script = Script(src1, path='../setup.py')
-        result = Script(src1, path='../setup.py').goto_definitions()
+        imports.load_module(script._evaluator, os.path.abspath(fname), src2)
+        result = script.goto_definitions()
         assert len(result) == 1
         assert result[0].description == 'class int'