detect buildout and add buildout eggs to sys.path

AUTHORS.txt

@@ -24,5 +24,6 @@ Steven Silvester (@blink1073)
 Colin Duquesnoy (@ColinDuquesnoy) <colin.duquesnoy@gmail.com>
 Jorgen Schaefer (@jorgenschaefer) <contact@jorgenschaefer.de>
 Fredrik Bergroth (@fbergroth)
+Mathias Fußenegger (@mfussenegger)
 
 Note: (@user) means a github user name.

jedi/evaluate/sys_path.py

@@ -3,6 +3,7 @@ import sys
 
 from jedi._compatibility import exec_function, unicode
 from jedi.parser import representation as pr
+from jedi.parser import Parser
 from jedi import debug
 from jedi import common
 
@@ -22,69 +23,97 @@ def get_sys_path():
     return [p for p in sys.path if p != ""]
 
 
+def _execute_code(module_path, code):
+    c = "import os; from os.path import *; result=%s"
+    variables = {'__file__': module_path}
+    try:
+        exec_function(c % code, variables)
+    except Exception:
+        debug.warning('sys.path manipulation detected, but failed to evaluate.')
+        return None
+    try:
+        res = variables['result']
+        if isinstance(res, str):
+            return os.path.abspath(res)
+        else:
+            return None
+    except KeyError:
+        return None
+
+
+def _paths_from_assignment(statement):
+    """
+    extracts the assigned strings from an assignment that looks as follows::
+
+    >>> sys.path[0:0] = ['module/path', 'another/module/path']
+    """
+
+    names = statement.get_defined_names()
+    if len(names) != 1:
+        return []
+    if [unicode(x) for x in names[0].names] != ['sys', 'path']:
+        return []
+    expressions = statement.expression_list()
+    if len(expressions) != 1 or not isinstance(expressions[0], pr.Array):
+        return
+    stmts = (s for s in expressions[0].values if isinstance(s, pr.Statement))
+    expression_lists = (s.expression_list() for s in stmts)
+    return [e.value for exprs in expression_lists for e in exprs
+            if isinstance(e, pr.Literal) and e.value]
+
+
+def _paths_from_insert(module_path, exe):
+    """ extract the inserted module path from an "sys.path.insert" statement
+    """
+    exe_type, exe.type = exe.type, pr.Array.NOARRAY
+    exe_pop = exe.values.pop(0)
+    res = _execute_code(module_path, exe.get_code())
+    exe.type = exe_type
+    exe.values.insert(0, exe_pop)
+    return res
+
+
+def _paths_from_call_expression(module_path, call):
+    """ extract the path from either "sys.path.append" or "sys.path.insert" """
+    if call.execution is None:
+        return
+    n = call.name
+    if not isinstance(n, pr.Name) or len(n.names) != 3:
+        return
+    names = [unicode(x) for x in n.names]
+    if names[:2] != ['sys', 'path']:
+        return
+    cmd = names[2]
+    exe = call.execution
+    if cmd == 'insert' and len(exe) == 2:
+        path = _paths_from_insert(module_path, exe)
+    elif cmd == 'append' and len(exe) == 1:
+        path = _execute_code(module_path, exe.get_code())
+    return path and [path] or []
+
+
+def _check_module(module):
+    try:
+        possible_stmts = module.used_names['path']
+    except KeyError:
+        return get_sys_path()
+    sys_path = list(get_sys_path())  # copy
+    statements = (p for p in possible_stmts if isinstance(p, pr.Statement))
+    for stmt in statements:
+        expressions = stmt.expression_list()
+        if len(expressions) == 1 and isinstance(expressions[0], pr.Call):
+            sys_path.extend(
+                _paths_from_call_expression(module.path, expressions[0]) or [])
+        elif (
+            hasattr(stmt, 'assignment_details') and
+            len(stmt.assignment_details) == 1
+        ):
+            sys_path.extend(_paths_from_assignment(stmt) or [])
+    return sys_path
+
+
 #@cache.memoize_default([]) TODO add some sort of cache again.
 def sys_path_with_modifications(module):
-    def execute_code(code):
-        c = "import os; from os.path import *; result=%s"
-        variables = {'__file__': module.path}
-        try:
-            exec_function(c % code, variables)
-        except Exception:
-            debug.warning('sys.path manipulation detected, but failed to evaluate.')
-            return None
-        try:
-            res = variables['result']
-            if isinstance(res, str):
-                return os.path.abspath(res)
-            else:
-                return None
-        except KeyError:
-            return None
-
-    def check_module(module):
-        try:
-            possible_stmts = module.used_names['path']
-        except KeyError:
-            return get_sys_path()
-
-        sys_path = list(get_sys_path())  # copy
-        for p in possible_stmts:
-            if not isinstance(p, pr.Statement):
-                continue
-            expression_list = p.expression_list()
-            # sys.path command is just one thing.
-            if len(expression_list) != 1 or not isinstance(expression_list[0], pr.Call):
-                continue
-            call = expression_list[0]
-            n = call.name
-            if not isinstance(n, pr.Name) or len(n.names) != 3:
-                continue
-            if [unicode(x) for x in n.names[:2]] != ['sys', 'path']:
-                continue
-            array_cmd = unicode(n.names[2])
-            if call.execution is None:
-                continue
-            exe = call.execution
-            if not (array_cmd == 'insert' and len(exe) == 2
-                    or array_cmd == 'append' and len(exe) == 1):
-                continue
-
-            if array_cmd == 'insert':
-                exe_type, exe.type = exe.type, pr.Array.NOARRAY
-                exe_pop = exe.values.pop(0)
-                res = execute_code(exe.get_code())
-                if res is not None:
-                    sys_path.insert(0, res)
-                    debug.dbg('sys path inserted: %s', res)
-                exe.type = exe_type
-                exe.values.insert(0, exe_pop)
-            elif array_cmd == 'append':
-                res = execute_code(exe.get_code())
-                if res is not None:
-                    sys_path.append(res)
-                    debug.dbg('sys path added: %s', res)
-        return sys_path
-
     if module.path is None:
         # Support for modules without a path is bad, therefore return the
         # normal path.
@@ -94,27 +123,82 @@ def sys_path_with_modifications(module):
     with common.ignored(OSError):
         os.chdir(os.path.dirname(module.path))
 
-    result = check_module(module)
+    result = _check_module(module)
     result += _detect_django_path(module.path)
-
+    # buildout scripts often contain the same sys.path modifications
+    # the set here is used to avoid duplicate sys.path entries
+    buildout_paths = set()
+    for module_path in _get_buildout_scripts(module.path):
+        try:
+            with open(module_path, 'rb') as f:
+                source = f.read()
+        except IOError:
+            pass
+        else:
+            p = Parser(common.source_to_unicode(source), module_path)
+            for path in _check_module(p.module):
+                if path not in buildout_paths:
+                    buildout_paths.add(path)
+                    result.append(path)
     # cleanup, back to old directory
     os.chdir(curdir)
-    return result
+    return list(result)
+
+
+def _traverse_parents(path):
+    while True:
+        new = os.path.dirname(path)
+        if new == path:
+            return
+        path = new
+        yield path
+
+
+def _get_parent_dir_with_file(path, filename):
+    for parent in _traverse_parents(path):
+        if os.path.isfile(os.path.join(parent, filename)):
+            return parent
+    return None
 
 
 def _detect_django_path(module_path):
     """ Detects the path of the very well known Django library (if used) """
     result = []
-    while True:
-        new = os.path.dirname(module_path)
-        # If the module_path doesn't change anymore, we're finished -> /
-        if new == module_path:
-            break
-        else:
-            module_path = new
 
+    for parent in _traverse_parents(module_path):
         with common.ignored(IOError):
-            with open(module_path + os.path.sep + 'manage.py'):
+            with open(parent + os.path.sep + 'manage.py'):
                 debug.dbg('Found django path: %s', module_path)
-                result.append(module_path)
+                result.append(parent)
     return result
+
+
+def _get_buildout_scripts(module_path):
+    """
+    if there is a 'buildout.cfg' file in one of the parent directories of the
+    given module it will return a list of all files in the buildout bin
+    directory that look like python files.
+
+    :param module_path: absolute path to the module.
+    :type module_path: str
+    """
+    project_root = _get_parent_dir_with_file(module_path, 'buildout.cfg')
+    if not project_root:
+        return []
+    bin_path = os.path.join(project_root, 'bin')
+    if not os.path.exists(bin_path):
+        return []
+    extra_module_paths = []
+    for filename in os.listdir(bin_path):
+        try:
+            filepath = os.path.join(bin_path, filename)
+            with open(filepath, 'r') as f:
+                firstline = f.readline()
+                if firstline.startswith('#!') and 'python' in firstline:
+                    extra_module_paths.append(filepath)
+        except IOError as e:
+            # either permission error or race cond. because file got deleted
+            # ignore
+            debug.warning(unicode(e))
+            continue
+    return extra_module_paths

test/test_evaluate/buildout_project/bin/app

@@ -0,0 +1,12 @@
+#!/usr/bin/python
+
+import sys
+sys.path[0:0] = [
+  '/usr/lib/python3.4/site-packages',
+  '/tmp/.buildout/eggs/important_package.egg'
+]
+
+import important_package
+
+if __name__ == '__main__':
+    sys.exit(important_package.main())

test/test_evaluate/test_buildout_detection.py

@@ -0,0 +1,71 @@
+import os
+from ..helpers import cwd_at
+from jedi._compatibility import u
+from jedi.parser import Parser
+from jedi.evaluate.sys_path import (
+    _get_parent_dir_with_file,
+    _get_buildout_scripts,
+    _check_module
+)
+
+
+@cwd_at('test/test_evaluate/buildout_project/src/proj_name')
+def test_parent_dir_with_file():
+    parent = _get_parent_dir_with_file(
+        os.path.abspath(os.curdir), 'buildout.cfg')
+    assert parent is not None
+    assert parent.endswith('test/test_evaluate/buildout_project')
+
+
+@cwd_at('test/test_evaluate/buildout_project/src/proj_name')
+def test_buildout_detection():
+    scripts = _get_buildout_scripts(os.path.abspath('./module_name.py'))
+    assert len(scripts) == 1
+    curdir = os.path.abspath(os.curdir)
+    appdir_path = os.path.normpath(os.path.join(curdir, '../../bin/app'))
+    assert scripts[0] == appdir_path
+
+
+def test_append_on_non_sys_path():
+    SRC = u("""
+class Dummy(object):
+    path = []
+
+d = Dummy()
+d.path.append('foo')""")
+    p = Parser(SRC)
+    paths = _check_module(p.module)
+    assert len(paths) > 0
+    assert 'foo' not in paths
+
+
+def test_path_from_invalid_sys_path_assignment():
+    SRC = u("""
+import sys
+sys.path = 'invalid'""")
+    p = Parser(SRC)
+    paths = _check_module(p.module)
+    assert len(paths) > 0
+    assert 'invalid' not in paths
+
+
+def test_path_from_sys_path_assignment():
+    SRC = u("""
+#!/usr/bin/python
+
+import sys
+sys.path[0:0] = [
+  '/usr/lib/python3.4/site-packages',
+  '/home/test/.buildout/eggs/important_package.egg'
+  ]
+
+path[0:0] = [1]
+
+import important_package
+
+if __name__ == '__main__':
+    sys.exit(important_package.main())""")
+    p = Parser(SRC)
+    paths = _check_module(p.module)
+    assert 1 not in paths
+    assert '/home/test/.buildout/eggs/important_package.egg' in paths