From bf43fcf1c60a458445cc303108aa641ef7bcc672 Mon Sep 17 00:00:00 2001 From: Mathias Fussenegger Date: Fri, 2 May 2014 12:02:21 +0200 Subject: [PATCH] detect buildout and add buildout eggs to sys.path --- AUTHORS.txt | 1 + jedi/evaluate/sys_path.py | 230 ++++++++++++------ test/test_evaluate/buildout_project/bin/app | 12 + .../buildout_project/bin/empty_file | 0 .../buildout_project/buildout.cfg | 0 .../src/proj_name/module_name.py | 0 test/test_evaluate/test_buildout_detection.py | 71 ++++++ 7 files changed, 241 insertions(+), 73 deletions(-) create mode 100644 test/test_evaluate/buildout_project/bin/app create mode 100644 test/test_evaluate/buildout_project/bin/empty_file create mode 100644 test/test_evaluate/buildout_project/buildout.cfg create mode 100644 test/test_evaluate/buildout_project/src/proj_name/module_name.py create mode 100644 test/test_evaluate/test_buildout_detection.py diff --git a/AUTHORS.txt b/AUTHORS.txt index 61b0e7a6..1a1d5903 100644 --- a/AUTHORS.txt +++ b/AUTHORS.txt @@ -24,5 +24,6 @@ Steven Silvester (@blink1073) Colin Duquesnoy (@ColinDuquesnoy) Jorgen Schaefer (@jorgenschaefer) Fredrik Bergroth (@fbergroth) +Mathias Fußenegger (@mfussenegger) Note: (@user) means a github user name. diff --git a/jedi/evaluate/sys_path.py b/jedi/evaluate/sys_path.py index 2390fd1b..d09cfec8 100644 --- a/jedi/evaluate/sys_path.py +++ b/jedi/evaluate/sys_path.py @@ -3,6 +3,7 @@ import sys from jedi._compatibility import exec_function, unicode from jedi.parser import representation as pr +from jedi.parser import Parser from jedi import debug from jedi import common @@ -22,69 +23,97 @@ def get_sys_path(): return [p for p in sys.path if p != ""] +def _execute_code(module_path, code): + c = "import os; from os.path import *; result=%s" + variables = {'__file__': module_path} + try: + exec_function(c % code, variables) + except Exception: + debug.warning('sys.path manipulation detected, but failed to evaluate.') + return None + try: + res = variables['result'] + if isinstance(res, str): + return os.path.abspath(res) + else: + return None + except KeyError: + return None + + +def _paths_from_assignment(statement): + """ + extracts the assigned strings from an assignment that looks as follows:: + + >>> sys.path[0:0] = ['module/path', 'another/module/path'] + """ + + names = statement.get_defined_names() + if len(names) != 1: + return [] + if [unicode(x) for x in names[0].names] != ['sys', 'path']: + return [] + expressions = statement.expression_list() + if len(expressions) != 1 or not isinstance(expressions[0], pr.Array): + return + stmts = (s for s in expressions[0].values if isinstance(s, pr.Statement)) + expression_lists = (s.expression_list() for s in stmts) + return [e.value for exprs in expression_lists for e in exprs + if isinstance(e, pr.Literal) and e.value] + + +def _paths_from_insert(module_path, exe): + """ extract the inserted module path from an "sys.path.insert" statement + """ + exe_type, exe.type = exe.type, pr.Array.NOARRAY + exe_pop = exe.values.pop(0) + res = _execute_code(module_path, exe.get_code()) + exe.type = exe_type + exe.values.insert(0, exe_pop) + return res + + +def _paths_from_call_expression(module_path, call): + """ extract the path from either "sys.path.append" or "sys.path.insert" """ + if call.execution is None: + return + n = call.name + if not isinstance(n, pr.Name) or len(n.names) != 3: + return + names = [unicode(x) for x in n.names] + if names[:2] != ['sys', 'path']: + return + cmd = names[2] + exe = call.execution + if cmd == 'insert' and len(exe) == 2: + path = _paths_from_insert(module_path, exe) + elif cmd == 'append' and len(exe) == 1: + path = _execute_code(module_path, exe.get_code()) + return path and [path] or [] + + +def _check_module(module): + try: + possible_stmts = module.used_names['path'] + except KeyError: + return get_sys_path() + sys_path = list(get_sys_path()) # copy + statements = (p for p in possible_stmts if isinstance(p, pr.Statement)) + for stmt in statements: + expressions = stmt.expression_list() + if len(expressions) == 1 and isinstance(expressions[0], pr.Call): + sys_path.extend( + _paths_from_call_expression(module.path, expressions[0]) or []) + elif ( + hasattr(stmt, 'assignment_details') and + len(stmt.assignment_details) == 1 + ): + sys_path.extend(_paths_from_assignment(stmt) or []) + return sys_path + + #@cache.memoize_default([]) TODO add some sort of cache again. def sys_path_with_modifications(module): - def execute_code(code): - c = "import os; from os.path import *; result=%s" - variables = {'__file__': module.path} - try: - exec_function(c % code, variables) - except Exception: - debug.warning('sys.path manipulation detected, but failed to evaluate.') - return None - try: - res = variables['result'] - if isinstance(res, str): - return os.path.abspath(res) - else: - return None - except KeyError: - return None - - def check_module(module): - try: - possible_stmts = module.used_names['path'] - except KeyError: - return get_sys_path() - - sys_path = list(get_sys_path()) # copy - for p in possible_stmts: - if not isinstance(p, pr.Statement): - continue - expression_list = p.expression_list() - # sys.path command is just one thing. - if len(expression_list) != 1 or not isinstance(expression_list[0], pr.Call): - continue - call = expression_list[0] - n = call.name - if not isinstance(n, pr.Name) or len(n.names) != 3: - continue - if [unicode(x) for x in n.names[:2]] != ['sys', 'path']: - continue - array_cmd = unicode(n.names[2]) - if call.execution is None: - continue - exe = call.execution - if not (array_cmd == 'insert' and len(exe) == 2 - or array_cmd == 'append' and len(exe) == 1): - continue - - if array_cmd == 'insert': - exe_type, exe.type = exe.type, pr.Array.NOARRAY - exe_pop = exe.values.pop(0) - res = execute_code(exe.get_code()) - if res is not None: - sys_path.insert(0, res) - debug.dbg('sys path inserted: %s', res) - exe.type = exe_type - exe.values.insert(0, exe_pop) - elif array_cmd == 'append': - res = execute_code(exe.get_code()) - if res is not None: - sys_path.append(res) - debug.dbg('sys path added: %s', res) - return sys_path - if module.path is None: # Support for modules without a path is bad, therefore return the # normal path. @@ -94,27 +123,82 @@ def sys_path_with_modifications(module): with common.ignored(OSError): os.chdir(os.path.dirname(module.path)) - result = check_module(module) + result = _check_module(module) result += _detect_django_path(module.path) - + # buildout scripts often contain the same sys.path modifications + # the set here is used to avoid duplicate sys.path entries + buildout_paths = set() + for module_path in _get_buildout_scripts(module.path): + try: + with open(module_path, 'rb') as f: + source = f.read() + except IOError: + pass + else: + p = Parser(common.source_to_unicode(source), module_path) + for path in _check_module(p.module): + if path not in buildout_paths: + buildout_paths.add(path) + result.append(path) # cleanup, back to old directory os.chdir(curdir) - return result + return list(result) + + +def _traverse_parents(path): + while True: + new = os.path.dirname(path) + if new == path: + return + path = new + yield path + + +def _get_parent_dir_with_file(path, filename): + for parent in _traverse_parents(path): + if os.path.isfile(os.path.join(parent, filename)): + return parent + return None def _detect_django_path(module_path): """ Detects the path of the very well known Django library (if used) """ result = [] - while True: - new = os.path.dirname(module_path) - # If the module_path doesn't change anymore, we're finished -> / - if new == module_path: - break - else: - module_path = new + for parent in _traverse_parents(module_path): with common.ignored(IOError): - with open(module_path + os.path.sep + 'manage.py'): + with open(parent + os.path.sep + 'manage.py'): debug.dbg('Found django path: %s', module_path) - result.append(module_path) + result.append(parent) return result + + +def _get_buildout_scripts(module_path): + """ + if there is a 'buildout.cfg' file in one of the parent directories of the + given module it will return a list of all files in the buildout bin + directory that look like python files. + + :param module_path: absolute path to the module. + :type module_path: str + """ + project_root = _get_parent_dir_with_file(module_path, 'buildout.cfg') + if not project_root: + return [] + bin_path = os.path.join(project_root, 'bin') + if not os.path.exists(bin_path): + return [] + extra_module_paths = [] + for filename in os.listdir(bin_path): + try: + filepath = os.path.join(bin_path, filename) + with open(filepath, 'r') as f: + firstline = f.readline() + if firstline.startswith('#!') and 'python' in firstline: + extra_module_paths.append(filepath) + except IOError as e: + # either permission error or race cond. because file got deleted + # ignore + debug.warning(unicode(e)) + continue + return extra_module_paths diff --git a/test/test_evaluate/buildout_project/bin/app b/test/test_evaluate/buildout_project/bin/app new file mode 100644 index 00000000..7394d2da --- /dev/null +++ b/test/test_evaluate/buildout_project/bin/app @@ -0,0 +1,12 @@ +#!/usr/bin/python + +import sys +sys.path[0:0] = [ + '/usr/lib/python3.4/site-packages', + '/tmp/.buildout/eggs/important_package.egg' +] + +import important_package + +if __name__ == '__main__': + sys.exit(important_package.main()) diff --git a/test/test_evaluate/buildout_project/bin/empty_file b/test/test_evaluate/buildout_project/bin/empty_file new file mode 100644 index 00000000..e69de29b diff --git a/test/test_evaluate/buildout_project/buildout.cfg b/test/test_evaluate/buildout_project/buildout.cfg new file mode 100644 index 00000000..e69de29b diff --git a/test/test_evaluate/buildout_project/src/proj_name/module_name.py b/test/test_evaluate/buildout_project/src/proj_name/module_name.py new file mode 100644 index 00000000..e69de29b diff --git a/test/test_evaluate/test_buildout_detection.py b/test/test_evaluate/test_buildout_detection.py new file mode 100644 index 00000000..a279c5ff --- /dev/null +++ b/test/test_evaluate/test_buildout_detection.py @@ -0,0 +1,71 @@ +import os +from ..helpers import cwd_at +from jedi._compatibility import u +from jedi.parser import Parser +from jedi.evaluate.sys_path import ( + _get_parent_dir_with_file, + _get_buildout_scripts, + _check_module +) + + +@cwd_at('test/test_evaluate/buildout_project/src/proj_name') +def test_parent_dir_with_file(): + parent = _get_parent_dir_with_file( + os.path.abspath(os.curdir), 'buildout.cfg') + assert parent is not None + assert parent.endswith('test/test_evaluate/buildout_project') + + +@cwd_at('test/test_evaluate/buildout_project/src/proj_name') +def test_buildout_detection(): + scripts = _get_buildout_scripts(os.path.abspath('./module_name.py')) + assert len(scripts) == 1 + curdir = os.path.abspath(os.curdir) + appdir_path = os.path.normpath(os.path.join(curdir, '../../bin/app')) + assert scripts[0] == appdir_path + + +def test_append_on_non_sys_path(): + SRC = u(""" +class Dummy(object): + path = [] + +d = Dummy() +d.path.append('foo')""") + p = Parser(SRC) + paths = _check_module(p.module) + assert len(paths) > 0 + assert 'foo' not in paths + + +def test_path_from_invalid_sys_path_assignment(): + SRC = u(""" +import sys +sys.path = 'invalid'""") + p = Parser(SRC) + paths = _check_module(p.module) + assert len(paths) > 0 + assert 'invalid' not in paths + + +def test_path_from_sys_path_assignment(): + SRC = u(""" +#!/usr/bin/python + +import sys +sys.path[0:0] = [ + '/usr/lib/python3.4/site-packages', + '/home/test/.buildout/eggs/important_package.egg' + ] + +path[0:0] = [1] + +import important_package + +if __name__ == '__main__': + sys.exit(important_package.main())""") + p = Parser(SRC) + paths = _check_module(p.module) + assert 1 not in paths + assert '/home/test/.buildout/eggs/important_package.egg' in paths