diff --git a/jedi/api/__init__.py b/jedi/api/__init__.py index b78a9d11..a2f4b89e 100644 --- a/jedi/api/__init__.py +++ b/jedi/api/__init__.py @@ -27,7 +27,7 @@ from jedi import keywords from jedi.api import classes from jedi.evaluate import Evaluator, filter_private_variable from jedi.evaluate import representation as er -from jedi.evaluate import builtin +from jedi.evaluate import compiled from jedi.evaluate import imports from jedi.evaluate import helpers @@ -137,20 +137,19 @@ class Script(object): path, dot, like = self._get_completion_parts() user_stmt = self._user_stmt(True) - bs = builtin.Builtin.scope - completions = get_completions(user_stmt, bs) + b = compiled.builtin + completions = get_completions(user_stmt, b) if not dot: # add named params for call_def in self.call_signatures(): - if not call_def.module.is_builtin(): + if not isinstance(call_def.module, compiled.PyObject): for p in call_def.params: completions.append((p.get_name(), p)) if not path and not isinstance(user_stmt, pr.Import): # add keywords - completions += ((k, bs) for k in keywords.keyword_names( - all=True)) + completions += ((k, b) for k in keywords.keyword_names(all=True)) needs_dot = not dot and path @@ -193,7 +192,7 @@ class Script(object): debug.dbg('possible scopes', scopes) for s in scopes: if s.isinstance(er.Function): - names = s.get_magic_method_names() + names = s.get_magic_function_names() else: if isinstance(s, imports.ImportPath): under = like + self._user_context.get_path_after_cursor() @@ -519,7 +518,8 @@ class Script(object): debug.speed('func_call followed') return [classes.CallDef(o, index, call) for o in origins - if o.isinstance(er.Function, er.Instance, er.Class)] + if o.isinstance(er.Function, er.Instance, er.Class) + or isinstance(o, compiled.PyObject) and o.type() != 'module'] def _func_call_and_param_index(self): debug.speed('func_call start') @@ -563,11 +563,10 @@ class Script(object): match = re.match(r'^(.*?)(\.|)(\w?[\w\d]*)$', path, flags=re.S) return match.groups() - @staticmethod - def _sorted_defs(d): + def _sorted_defs(self, d): # Note: `or ''` below is required because `module_path` could be # None and you can't compare None and str in Python 3. - return sorted(d, key=lambda x: (x.module_path or '', x.line, x.column)) + return sorted(d, key=lambda x: (x.module_path or '', x.line or 0, x.column or 0)) class Interpreter(Script): diff --git a/jedi/api/classes.py b/jedi/api/classes.py index 341882f2..8549e121 100644 --- a/jedi/api/classes.py +++ b/jedi/api/classes.py @@ -14,6 +14,7 @@ from jedi import cache from jedi.evaluate import representation as er from jedi.evaluate import iterable from jedi.evaluate import imports +from jedi.evaluate import compiled from jedi import keywords @@ -70,8 +71,11 @@ class BaseDefinition(object): # generate a path to the definition self._module = definition.get_parent_until() - self.module_path = self._module.path - """Shows the file path of a module. e.g. ``/usr/lib/python2.7/os.py``""" + if self.in_builtin_module(): + self.module_path = None + else: + self.module_path = self._module.path + """Shows the file path of a module. e.g. ``/usr/lib/python2.7/os.py``""" @property def start_pos(self): @@ -134,8 +138,10 @@ class BaseDefinition(object): """ # generate the type stripped = self._definition - if isinstance(self._definition, er.InstanceElement): - stripped = self._definition.var + if isinstance(stripped, compiled.PyObject): + return stripped.type() + if isinstance(stripped, er.InstanceElement): + stripped = stripped.var if isinstance(stripped, pr.Name): stripped = stripped.parent return type(stripped).__name__.lower() @@ -167,18 +173,17 @@ class BaseDefinition(object): The module name. >>> from jedi import Script - >>> source = 'import datetime' - >>> script = Script(source, 1, len(source), 'example.py') + >>> source = 'import json' + >>> script = Script(source, path='example.py') >>> d = script.goto_definitions()[0] >>> print(d.module_name) # doctest: +ELLIPSIS - datetime + json """ return str(self._module.name) def in_builtin_module(self): """Whether this is a builtin module.""" - return not (self.module_path is None or - self.module_path.endswith('.py')) + return isinstance(self._module, compiled.PyObject) @property def line_nr(self): @@ -232,7 +237,7 @@ class BaseDefinition(object): """ try: - return self._definition.doc + return self._definition.doc or '' # Always a String, never None. except AttributeError: return self.raw_doc @@ -435,7 +440,9 @@ class Definition(BaseDefinition): if isinstance(d, er.InstanceElement): d = d.var - if isinstance(d, pr.Name): + if isinstance(d, compiled.PyObject): + return d.name + elif isinstance(d, pr.Name): return d.names[-1] if d.names else None elif isinstance(d, iterable.Array): return unicode(d.type) @@ -454,7 +461,6 @@ class Definition(BaseDefinition): return d.assignment_details[0][1].values[0][0].name.names[-1] except IndexError: return None - return None @property def description(self): @@ -490,7 +496,9 @@ class Definition(BaseDefinition): if isinstance(d, pr.Name): d = d.parent - if isinstance(d, iterable.Array): + if isinstance(d, compiled.PyObject): + d = d.type() + ' ' + d.name + elif isinstance(d, iterable.Array): d = 'class ' + d.type elif isinstance(d, (pr.Class, er.Class, er.Instance)): d = 'class ' + unicode(d.name) @@ -518,12 +526,7 @@ class Definition(BaseDefinition): .. todo:: Add full path. This function is should return a `module.class.function` path. """ - if self.module_path.endswith('.py') \ - and not isinstance(self._definition, pr.Module): - position = '@%s' % (self.line) - else: - # is a builtin or module - position = '' + position = '' if self.in_builtin_module else '@%s' % (self.line) return "%s:%s%s" % (self.module_name, self.description, position) def defined_names(self): diff --git a/jedi/debug.py b/jedi/debug.py index d7861021..6ab6ff6b 100644 --- a/jedi/debug.py +++ b/jedi/debug.py @@ -4,13 +4,13 @@ import os import time try: - if not os.name == 'nt': + if os.name == 'nt': + # does not work on Windows, as pyreadline and colorama interfere + raise ImportError + else: # Use colorama for nicer console output. from colorama import Fore, init init() - # does not work on Windows, as pyreadline and colorama interfere - else: - raise ImportError except ImportError: class Fore(object): RED = '' @@ -29,11 +29,24 @@ enable_notice = False # callback, interface: level, str debug_function = None ignored_modules = ['jedi.evaluate.builtin', 'jedi.parser'] +debug_indent = -1 def reset_time(): - global start_time + global start_time, debug_indent start_time = time.time() + debug_indent = -1 + + +def increase_indent(func): + """Decorator for makin """ + def wrapper(*args, **kwargs): + global debug_indent + debug_indent += 1 + result = func(*args, **kwargs) + debug_indent -= 1 + return result + return wrapper def dbg(*args): @@ -42,18 +55,21 @@ def dbg(*args): frm = inspect.stack()[1] mod = inspect.getmodule(frm[0]) if not (mod.__name__ in ignored_modules): - debug_function(NOTICE, 'dbg: ' + ', '.join(u(a) for a in args)) + i = ' ' * debug_indent + debug_function(NOTICE, i + 'dbg: ' + ', '.join(u(a) for a in args)) def warning(*args): if debug_function and enable_warning: - debug_function(WARNING, 'warning: ' + ', '.join(u(a) for a in args)) + i = ' ' * debug_indent + debug_function(WARNING, i + 'warning: ' + ', '.join(u(a) for a in args)) def speed(name): if debug_function and enable_speed: now = time.time() - debug_function(SPEED, 'speed: ' + '%s %s' % (name, now - start_time)) + i = ' ' * debug_indent + debug_function(SPEED, i + 'speed: ' + '%s %s' % (name, now - start_time)) def print_to_stdout(level, str_out): diff --git a/jedi/evaluate/__init__.py b/jedi/evaluate/__init__.py index f08685b0..a1ce372a 100644 --- a/jedi/evaluate/__init__.py +++ b/jedi/evaluate/__init__.py @@ -76,13 +76,13 @@ from jedi import common from jedi.parser import representation as pr from jedi import debug from jedi.evaluate import representation as er -from jedi.evaluate import builtin from jedi.evaluate import imports from jedi.evaluate import recursion from jedi.evaluate import iterable from jedi.evaluate.cache import memoize_default from jedi.evaluate import stdlib from jedi.evaluate import finder +from jedi.evaluate import compiled class Evaluator(object): @@ -128,7 +128,7 @@ class Evaluator(object): true (default). >>> pairs[2] #doctest: +ELLIPSIS - (, [, ...]) + (, [, ...]) :rtype: [(pr.Scope, [pr.Name])] :return: Return an generator that yields a pair of scope and names. @@ -146,7 +146,9 @@ class Evaluator(object): if not (scope != non_flow and scope.isinstance(pr.Class) or scope.isinstance(pr.Flow) or scope.isinstance(er.Instance) - and non_flow.isinstance(er.Function)): + and non_flow.isinstance(er.Function) + or isinstance(scope, compiled.PyObject) + and scope.type() == 'class' and in_func_scope != scope): try: if isinstance(scope, er.Instance): for g in scope.scope_generator(): @@ -173,8 +175,7 @@ class Evaluator(object): # Add builtins to the global scope. if include_builtin: - builtin_scope = builtin.Builtin.scope - yield builtin_scope, builtin_scope.get_defined_names() + yield compiled.builtin, compiled.builtin.get_defined_names() def find_types(self, scope, name_str, position=None, search_global=False, is_goto=False, resolve_decorator=True): @@ -194,15 +195,15 @@ class Evaluator(object): @memoize_default(default=(), evaluator_is_first_arg=True) @recursion.recursion_decorator + @debug.increase_indent def eval_statement(self, stmt, seek_name=None): """ - The starting point of the completion. A statement always owns a call list, - which are the calls, that a statement does. - In case multiple names are defined in the statement, `seek_name` returns - the result for this name. + The starting point of the completion. A statement always owns a call + list, which are the calls, that a statement does. In case multiple + names are defined in the statement, `seek_name` returns the result for + this name. :param stmt: A `pr.Statement`. - :param seek_name: A string. """ debug.dbg('eval_statement %s (%s)' % (stmt, seek_name)) expression_list = stmt.expression_list() @@ -266,6 +267,8 @@ class Evaluator(object): er.Function, er.Class, er.Instance, iterable.ArrayInstance): result.append(call) # The string tokens are just operations (+, -, etc.) + elif isinstance(call, compiled.PyObject): + result.append(call) elif not isinstance(call, (str, unicode)): if isinstance(call, pr.Call) and str(call.name) == 'if': # Ternary operators. @@ -281,8 +284,8 @@ class Evaluator(object): result += self.eval_call(call) elif call == '*': if [r for r in result if isinstance(r, iterable.Array) - or isinstance(r, er.Instance) - and str(r.name) == 'str']: + or isinstance(r, compiled.PyObject) + and isinstance(r.obj, (str, unicode))]: # if it is an iterable, ignore * operations next(calls_iterator) return set(result) @@ -308,16 +311,12 @@ class Evaluator(object): else: if isinstance(current, pr.NamePart): # This is the first global lookup. - scopes = self.find_types(scope, current, position=position, - search_global=True) + types = self.find_types(scope, current, position=position, + search_global=True) else: # for pr.Literal - scopes = self.find_types(builtin.Builtin.scope, current.type_as_string()) - # Make instances of those number/string objects. - scopes = itertools.chain.from_iterable( - self.execute(s, (current.value,)) for s in scopes - ) - types = imports.strip_imports(self, scopes) + types = [compiled.create(current.value)] + types = imports.strip_imports(self, types) return self.follow_path(path, types, scope, position=position) @@ -332,8 +331,8 @@ class Evaluator(object): results_new = [] iter_paths = itertools.tee(path, len(types)) - for i, type in enumerate(types): - fp = self._follow_path(iter_paths[i], type, call_scope, position=position) + for i, typ in enumerate(types): + fp = self._follow_path(iter_paths[i], typ, call_scope, position=position) if fp is not None: results_new += fp else: @@ -341,7 +340,7 @@ class Evaluator(object): return types return results_new - def _follow_path(self, path, type, scope, position=None): + def _follow_path(self, path, typ, scope, position=None): """ Uses a generator and tries to complete the path, e.g.:: @@ -355,52 +354,58 @@ class Evaluator(object): current = next(path) except StopIteration: return None - debug.dbg('_follow_path: %s in scope %s' % (current, type)) + debug.dbg('_follow_path: %s in scope %s' % (current, typ)) result = [] if isinstance(current, pr.Array): # This must be an execution, either () or []. if current.type == pr.Array.LIST: - if hasattr(type, 'get_index_types'): - result = type.get_index_types(current) + if hasattr(typ, 'get_index_types'): + result = typ.get_index_types(current) elif current.type not in [pr.Array.DICT]: # Scope must be a class or func - make an instance or execution. - debug.dbg('exe', type) - result = self.execute(type, current) + result = self.execute(typ, current) else: # Curly braces are not allowed, because they make no sense. - debug.warning('strange function call with {}', current, type) + debug.warning('strange function call with {}', current, typ) else: # The function must not be decorated with something else. - if type.isinstance(er.Function): - type = type.get_magic_method_scope() + if typ.isinstance(er.Function): + typ = typ.get_magic_function_scope() else: # This is the typical lookup while chaining things. - if filter_private_variable(type, scope, current): + if filter_private_variable(typ, scope, current): return [] - result = imports.strip_imports(self, self.find_types(type, current, - position=position)) + types = self.find_types(typ, current, position=position) + result = imports.strip_imports(self, types) return self.follow_path(path, set(result), scope, position=position) + @debug.increase_indent def execute(self, obj, params=(), evaluate_generator=False): if obj.isinstance(er.Function): obj = obj.get_decorated_func() + debug.dbg('execute:', obj, params) try: return stdlib.execute(self, obj, params) except stdlib.NotInStdLib: pass - if obj.isinstance(er.Class): + if obj.isinstance(compiled.PyObject): + if obj.is_executable_class(): + return [er.Instance(self, obj, params)] + else: + return list(obj.execute_function(self, params)) + elif obj.isinstance(er.Class): # There maybe executions of executions. return [er.Instance(self, obj, params)] elif isinstance(obj, iterable.Generator): return obj.iter_content() else: stmts = [] - try: - obj.returns # Test if it is a function - except AttributeError: + if obj.isinstance(er.Function): + stmts = er.FunctionExecution(self, obj, params).get_return_types(evaluate_generator) + else: if hasattr(obj, 'execute_subscope_by_name'): try: stmts = obj.execute_subscope_by_name('__call__', params) @@ -408,10 +413,8 @@ class Evaluator(object): debug.warning("no __call__ func available", obj) else: debug.warning("no execution possible", obj) - else: - stmts = er.FunctionExecution(self, obj, params).get_return_types(evaluate_generator) - debug.dbg('execute: %s in %s' % (stmts, self)) + debug.dbg('execute result: %s in %s' % (stmts, obj)) return imports.strip_imports(self, stmts) def goto(self, stmt, call_path=None): @@ -450,9 +453,14 @@ def filter_private_variable(scope, call_scope, var_name): """private variables begin with a double underline `__`""" if isinstance(var_name, (str, unicode)) and isinstance(scope, er.Instance)\ and var_name.startswith('__') and not var_name.endswith('__'): - s = call_scope.get_parent_until((pr.Class, er.Instance)) - if s != scope and s != scope.base.base: - return True + s = call_scope.get_parent_until((pr.Class, er.Instance, compiled.PyObject)) + if s != scope: + if isinstance(scope.base, compiled.PyObject): + if s != scope.base: + return True + else: + if s != scope.base.base: + return True return False diff --git a/jedi/evaluate/builtin.py b/jedi/evaluate/builtin.py deleted file mode 100644 index 19e0147c..00000000 --- a/jedi/evaluate/builtin.py +++ /dev/null @@ -1,455 +0,0 @@ -""" -A big part of the Python standard libraries are unfortunately not only written -in Python. The process works like this: - -- ``BuiltinModule`` imports the builtin module (e.g. ``sys``) -- then ``BuiltinModule`` generates code with the docstrings of builtin - functions. -- The :mod:`parsing` parser processes the generated code. - -This is possible, because many builtin functions supply docstrings, for example -the method ``list.index`` has the following attribute ``__doc__``: - - L.index(value, [start, [stop]]) -> integer -- return first index of value. - Raises ValueError if the value is not present. - -`PEP 257 `_ -teaches how docstrings should look like for C functions. - -Additionally there's a ``Builtin`` instance in this module, to make it -possible to access functions like ``list`` and ``int`` directly, the same way -|jedi| access other functions. -""" - -from jedi._compatibility import exec_function, is_py3k - -import re -import sys -import os -if is_py3k: - import io -import types -import inspect - -from jedi import common -from jedi import debug -from jedi.parser import Parser -from jedi.parser import fast -from jedi.evaluate.sys_path import get_sys_path -from jedi import cache - - -class BuiltinModule(object): - """ - This module is a parser for all builtin modules, which are programmed in - C/C++. It should also work on third party modules. - It can be instantiated with either a path or a name of the module. The path - is important for third party modules. - - :param name: The name of the module. - :param path: The path of the module. - :param sys_path: The sys.path, which is can be customizable. - """ - - map_types = { - 'floating point number': '0.0', - 'string': '""', - 'str': '""', - 'character': '"a"', - 'integer': '0', - 'int': '0', - 'dictionary': '{}', - 'list': '[]', - 'file object': 'file("")', - # TODO things like dbg: ('not working', 'tuple of integers') - } - - if is_py3k: - map_types['file object'] = 'import io; return io.TextIOWrapper()' - - def __init__(self, path=None, name=None, sys_path=None): - if sys_path is None: - sys_path = get_sys_path() - self.sys_path = list(sys_path) - - if not name: - name = os.path.basename(path) - name = name.rpartition('.')[0] # cut file type (normally .so) - self.name = name - - self.path = path and os.path.abspath(path) - - @property - @cache.underscore_memoization - def parser(self): - """ get the parser lazy """ - return cache.load_parser(self.path, self.name) or self._load_module() - - def _load_module(self): - source = _generate_code(self.module, self._load_mixins()) - p = self.path or self.name - p = fast.FastParser(source, p) - cache.save_parser(self.path, self.name, p) - return p - - @property - @cache.underscore_memoization - def module(self): - """get module also lazy""" - def load_module(name, path): - if path: - self.sys_path.insert(0, path) - - temp, sys.path = sys.path, self.sys_path - content = {} - try: - exec_function('import %s as module' % name, content) - module = content['module'] - except AttributeError: - # use sys.modules, because you cannot access some modules - # directly. -> #59 - module = sys.modules[name] - sys.path = temp - - if path: - self.sys_path.pop(0) - return module - - # module might already be defined - path = self.path - name = self.name - if self.path: - dot_path = [] - p = self.path - # search for the builtin with the correct path - while p and p not in sys.path: - p, sep, mod = p.rpartition(os.path.sep) - dot_path.append(mod.partition('.')[0]) - if p: - name = ".".join(reversed(dot_path)) - path = p - else: - path = os.path.dirname(self.path) - return load_module(name, path) - - def _load_mixins(self): - """ - Load functions that are mixed in to the standard library. - E.g. builtins are written in C (binaries), but my autocompletion only - understands Python code. By mixing in Python code, the autocompletion - should work much better for builtins. - """ - regex = r'^(def|class)\s+([\w\d]+)' - - def process_code(code, depth=0): - funcs = {} - matches = list(re.finditer(regex, code, re.MULTILINE)) - positions = [m.start() for m in matches] - for i, pos in enumerate(positions): - try: - code_block = code[pos:positions[i + 1]] - except IndexError: - code_block = code[pos:len(code)] - structure_name = matches[i].group(1) - name = matches[i].group(2) - if structure_name == 'def': - funcs[name] = code_block - elif structure_name == 'class': - if depth > 0: - raise NotImplementedError() - - # remove class line - c = re.sub(r'^[^\n]+', '', code_block) - # remove whitespace - c = re.compile(r'^[ ]{4}', re.MULTILINE).sub('', c) - - funcs[name] = process_code(c) - else: - raise NotImplementedError() - return funcs - - name = self.name - # sometimes there are stupid endings like `_sqlite3.cpython-32mu` - name = re.sub(r'\..*', '', name) - - if name == '__builtin__' and not is_py3k: - name = 'builtins' - path = os.path.dirname(os.path.abspath(__file__)) - try: - with open(os.path.join(path, 'mixin', name) + '.pym') as f: - s = f.read() - except IOError: - return {} - else: - mixin_dct = process_code(s) - if is_py3k and self.name == Builtin.name: - # in the case of Py3k xrange is now range - mixin_dct['range'] = mixin_dct['xrange'] - return mixin_dct - - -def _generate_code(scope, mixin_funcs={}, depth=0): - """ - Generate a string, which uses python syntax as an input to the Parser. - """ - def get_doc(obj, indent=False): - doc = inspect.getdoc(obj) - if doc: - doc = ('r"""\n%s\n"""\n' % doc) - if indent: - doc = common.indent_block(doc) - return doc - return '' - - def is_in_base_classes(cls, name, comparison): - """ Base classes may contain the exact same object """ - if name in mixin_funcs: - return False - try: - mro = cls.mro() - except TypeError: - # this happens, if cls == type - return False - for base in mro[1:]: - try: - attr = getattr(base, name) - except AttributeError: - continue - if attr == comparison: - return True - return False - - def get_scope_objects(names): - """ - Looks for the names defined with dir() in an objects and divides - them into different object types. - """ - classes = {} - funcs = {} - stmts = {} - members = {} - for n in names: - try: - # this has a builtin_function_or_method - exe = getattr(scope, n) - except AttributeError: - # happens e.g. in properties of - # PyQt4.QtGui.QStyleOptionComboBox.currentText - # -> just set it to None - members[n] = None - else: - if inspect.isclass(scope): - if is_in_base_classes(scope, n, exe): - continue - if inspect.isbuiltin(exe) or inspect.ismethod(exe) \ - or inspect.ismethoddescriptor(exe): - funcs[n] = exe - elif inspect.isclass(exe) or inspect.ismodule(exe): - classes[n] = exe - elif inspect.ismemberdescriptor(exe): - members[n] = exe - else: - stmts[n] = exe - return classes, funcs, stmts, members - - code = '' - if inspect.ismodule(scope): # generate comment where the code's from. - try: - path = scope.__file__ - except AttributeError: - path = '?' - code += '# Generated module %s from %s\n' % (scope.__name__, path) - - code += get_doc(scope) - - # Remove some magic vars, (TODO why?) - names = set(dir(scope)) - set(['__file__', '__name__', '__doc__', - '__path__', '__package__']) - - classes, funcs, stmts, members = get_scope_objects(names) - - # classes - for name, cl in classes.items(): - bases = (c.__name__ for c in cl.__bases__) if inspect.isclass(cl) \ - else [] - code += 'class %s(%s):\n' % (name, ','.join(bases)) - if depth == 0: - try: - mixin = mixin_funcs[name] - except KeyError: - mixin = {} - cl_code = _generate_code(cl, mixin, depth + 1) - code += common.indent_block(cl_code) - code += '\n' - - # functions - for name, func in funcs.items(): - params, ret = _parse_function_doc(func) - if depth > 0: - params = 'self, ' + params - doc_str = get_doc(func, indent=True) - try: - mixin = mixin_funcs[name] - except KeyError: - # normal code generation - code += 'def %s(%s):\n' % (name, params) - code += doc_str - code += common.indent_block('%s\n\n' % ret) - else: - # generation of code with mixins - # the parser only supports basic functions with a newline after - # the double dots - # find doc_str place - try: - pos = re.search(r'\):\s*\n', mixin).end() - except TypeError: - # pypy uses a different reversed builtin - if name == 'reversed': - mixin = 'def reversed(sequence):\n' \ - ' for i in self.__sequence: yield i' - pos = 24 - else: - debug.warning('mixin trouble in pypy: %s', name) - raise - if pos is None: - raise Exception("Builtin function not parsed correctly") - code += mixin[:pos] + doc_str + mixin[pos:] - - # class members (functions) properties? - for name, func in members.items(): - # recursion problem in properties TODO remove - if name in ['fget', 'fset', 'fdel']: - continue - ret = 'pass' - code += '@property\ndef %s(self):\n' % (name) - code += common.indent_block(get_doc(func) + '%s\n\n' % ret) - - # variables - for name, value in stmts.items(): - if is_py3k: - file_type = io.TextIOWrapper - else: - file_type = types.FileType - if isinstance(value, file_type): - value = 'open()' - elif name == 'None': - value = '' - elif type(value).__name__ in ['int', 'bool', 'float', - 'dict', 'list', 'tuple']: - value = repr(value) - else: - # get the type, if the type is not simple. - mod = type(value).__module__ - value = type(value).__name__ + '()' - if mod != '__builtin__': - value = '%s.%s' % (mod, value) - code += '%s = %s\n' % (name, value) - - return code - - -def _parse_function_doc(func): - """ - Takes a function and returns the params and return value as a tuple. - This is nothing more than a docstring parser. - """ - # TODO: things like utime(path, (atime, mtime)) and a(b [, b]) -> None - doc = inspect.getdoc(func) - - if doc is None: - return '', 'pass' - - # get full string, parse round parentheses: def func(a, (b,c)) - try: - # unbound methods such as pyqtSignals have no __name__ - if not hasattr(func, "__name__"): - return '', 'pass' - count = 0 - debug.dbg(func, func.__name__, doc) - start = doc.index('(') - for i, s in enumerate(doc[start:]): - if s == '(': - count += 1 - elif s == ')': - count -= 1 - if count == 0: - end = start + i - break - param_str = doc[start + 1:end] - except (ValueError, UnboundLocalError): - # ValueError for doc.index - # UnboundLocalError for undefined end in last line - debug.dbg('no brackets found - no param') - end = 0 - param_str = '' - else: - # remove square brackets, that show an optional param ( = None) - def change_options(m): - args = m.group(1).split(',') - for i, a in enumerate(args): - if a and '=' not in a: - args[i] += '=None' - return ','.join(args) - - while True: - param_str, changes = re.subn(r' ?\[([^\[\]]+)\]', - change_options, param_str) - if changes == 0: - break - param_str = param_str.replace('-', '_') # see: isinstance.__doc__ - - # parse return value - r = re.search('-[>-]* ', doc[end:end + 7]) - if r is None: - ret = '' - else: - index = end + r.end() - # get result type, which can contain newlines - pattern = re.compile(r'(,\n|[^\n-])+') - ret_str = pattern.match(doc, index).group(0).strip() - # New object -> object() - ret_str = re.sub(r'[nN]ew (.*)', r'\1()', ret_str) - - ret = BuiltinModule.map_types.get(ret_str, ret_str) - if ret == ret_str and ret not in ['None', 'object', 'tuple', 'set']: - debug.dbg('not working', ret_str) - - ret = ('return ' if 'return' not in ret else '') + ret - return param_str, ret - - -class Builtin(object): - """ The builtin scope / module """ - # Python 3 compatibility - if is_py3k: - name = 'builtins' - else: - name = '__builtin__' - - @property - @cache.underscore_memoization - def builtin(self): - return BuiltinModule(name=self.name) - - @property - def scope(self): - return self.builtin.parser.module - - def magic_function_scope(self, evaluator): - try: - return self._magic_function_scope - except AttributeError: - # depth = 1 because this is not a module - class Container(object): - FunctionType = types.FunctionType - source = _generate_code(Container, depth=0) - parser = Parser(source, None) - module = parser.module - module.parent = self.scope - typ = evaluator.follow_path(iter(['FunctionType']), [module], module) - - s = self._magic_function_scope = typ.pop() - return s - - -Builtin = Builtin() diff --git a/jedi/evaluate/compiled/__init__.py b/jedi/evaluate/compiled/__init__.py new file mode 100644 index 00000000..fa65e660 --- /dev/null +++ b/jedi/evaluate/compiled/__init__.py @@ -0,0 +1,302 @@ +""" +Imitate the parser representation. +""" +import inspect +import re +import sys +import os + +from jedi._compatibility import builtins as _builtins, exec_function +from jedi import debug +from jedi.parser.representation import Base +from jedi.cache import underscore_memoization +from jedi.evaluate.sys_path import get_sys_path +from . import fake + + +class PyObject(Base): + # comply with the parser + start_pos = 0, 0 + asserts = [] + path = None # modules have this attribute - set it to None. + + def __init__(self, obj, parent=None): + self.obj = obj + self.parent = parent + self.doc = inspect.getdoc(obj) + + def __repr__(self): + return '<%s: %s>' % (type(self).__name__, self.obj) + + def get_parent_until(self, *args, **kwargs): + # compiled modules only use functions and classes/methods (2 levels) + return getattr(self.parent, 'parent', self.parent) or self.parent or self + + @underscore_memoization + def _parse_function_doc(self): + if self.doc is None: + return '', '' + + return _parse_function_doc(self.doc) + + def type(self): + cls = self._cls().obj + if inspect.isclass(cls): + return 'class' + elif inspect.ismodule(cls): + return 'module' + elif inspect.isbuiltin(cls) or inspect.ismethod(cls) \ + or inspect.ismethoddescriptor(cls): + return 'def' + + def is_executable_class(self): + return inspect.isclass(self.obj) + + @underscore_memoization + def _cls(self): + # Ensures that a PyObject is returned that is not an instance (like list) + if fake.is_class_instance(self.obj): + try: + c = self.obj.__class__ + except AttributeError: + # happens with numpy.core.umath._UFUNC_API (you get it + # automatically by doing `import numpy`. + c = type(None) + return PyObject(c, self.parent) + return self + + def get_defined_names(self): + cls = self._cls() + for name in dir(cls.obj): + yield PyName(cls, name) + + def instance_names(self): + # TODO REMOVE (temporary until the Instance method is removed) + return self.get_defined_names() + + def get_subscope_by_name(self, name): + if name in dir(self._cls().obj): + return PyName(self._cls(), name).parent + else: + raise KeyError("CompiledObject doesn't have an attribute '%s'." % name) + + @property + def name(self): + # might not exist sometimes (raises AttributeError) + return self._cls().obj.__name__ + + def execute_function(self, evaluator, params): + if self.type() != 'def': + return + for name in self._parse_function_doc()[1].split(): + try: + bltn_obj = _create_from_name(builtin, builtin, name) + except AttributeError: + continue + else: + if isinstance(bltn_obj, PyObject): + yield bltn_obj + else: + for result in evaluator.execute(bltn_obj, params): + yield result + + @property + @underscore_memoization + def subscopes(self): + """ + Returns only the faked scopes - the other ones are not important for + internal analysis. + """ + module = self.get_parent_until() + faked_subscopes = [] + for name in dir(self._cls().obj): + f = fake.get_faked(module.obj, self.obj, name) + if f: + f.parent = self + faked_subscopes.append(f) + return faked_subscopes + + def get_self_attributes(self): + return [] # Instance compatibility + + def get_imports(self): + return [] # Builtins don't have imports + + +class PyName(object): + def __init__(self, obj, name): + self._obj = obj + self.name = name + self.start_pos = 0, 0 # an illegal start_pos, to make sorting easy. + + def get_parent_until(self): + return self.parent.get_parent_until() + + def __str__(self): + return self.name + + def __repr__(self): + return '<%s: (%s).%s>' % (type(self).__name__, self._obj.name, self.name) + + @property + @underscore_memoization + def parent(self): + module = self._obj.get_parent_until() + return _create_from_name(module, self._obj, self.name) + + @property + def names(self): + return [self.name] # compatibility with parser.representation.Name + + def get_code(self): + return self.name + + +def load_module(path, name): + if not name: + name = os.path.basename(path) + name = name.rpartition('.')[0] # cut file type (normally .so) + + # sometimes there are endings like `_sqlite3.cpython-32mu` + name = re.sub(r'\..*', '', name) + + if path: + dot_path = [] + p = path + # if path is not in sys.path, we need to make a well defined import + # like `from numpy.core import umath.` + while p and p not in sys.path: + p, sep, mod = p.rpartition(os.path.sep) + dot_path.insert(0, mod.partition('.')[0]) + if p: + name = ".".join(dot_path) + path = p + else: + path = os.path.dirname(path) + + sys_path = get_sys_path() + if path: + sys_path.insert(0, path) + + temp, sys.path = sys.path, sys_path + content = {} + try: + exec_function('import %s as module' % name, content) + module = content['module'] + except AttributeError: + # use sys.modules, because you cannot access some modules + # directly. -> github issue #59 + module = sys.modules[name] + sys.path = temp + return PyObject(module) + + +docstr_defaults = { + 'floating point number': 'float', + 'character': 'str', + 'integer': 'int', + 'dictionary': 'dict', + 'string': 'str', +} + + +def _parse_function_doc(doc): + """ + Takes a function and returns the params and return value as a tuple. + This is nothing more than a docstring parser. + + TODO docstrings like utime(path, (atime, mtime)) and a(b [, b]) -> None + TODO docstrings like 'tuple of integers' + """ + # parse round parentheses: def func(a, (b,c)) + try: + count = 0 + start = doc.index('(') + for i, s in enumerate(doc[start:]): + if s == '(': + count += 1 + elif s == ')': + count -= 1 + if count == 0: + end = start + i + break + param_str = doc[start + 1:end] + except (ValueError, UnboundLocalError): + # ValueError for doc.index + # UnboundLocalError for undefined end in last line + debug.dbg('no brackets found - no param') + end = 0 + param_str = '' + else: + # remove square brackets, that show an optional param ( = None) + def change_options(m): + args = m.group(1).split(',') + for i, a in enumerate(args): + if a and '=' not in a: + args[i] += '=None' + return ','.join(args) + + while True: + param_str, changes = re.subn(r' ?\[([^\[\]]+)\]', + change_options, param_str) + if changes == 0: + break + param_str = param_str.replace('-', '_') # see: isinstance.__doc__ + + # parse return value + r = re.search('-[>-]* ', doc[end:end + 7]) + if r is None: + ret = '' + else: + index = end + r.end() + # get result type, which can contain newlines + pattern = re.compile(r'(,\n|[^\n-])+') + ret_str = pattern.match(doc, index).group(0).strip() + # New object -> object() + ret_str = re.sub(r'[nN]ew (.*)', r'\1()', ret_str) + + ret = docstr_defaults.get(ret_str, ret_str) + + return param_str, ret + + +class Builtin(PyObject): + def get_defined_names(self): + # Filter None, because it's really just a keyword, nobody wants to + # access it. + return [d for d in super(Builtin, self).get_defined_names() if d.name != 'None'] + + +builtin = Builtin(_builtins) +magic_function_class = PyObject(type(load_module), parent=builtin) + + +def _create_from_name(module, parent, name): + faked = fake.get_faked(module.obj, parent.obj, name) + # only functions are necessary. + if faked is not None: + faked.parent = parent + return faked + + try: + obj = getattr(parent.obj, name) + except AttributeError: + # happens e.g. in properties of + # PyQt4.QtGui.QStyleOptionComboBox.currentText + # -> just set it to None + obj = None + return PyObject(obj, parent) + + +def create(obj, parent=builtin, module=None): + """ + A very weird interface class to this module. The more options provided the + more acurate loading compiled objects is. + """ + if not inspect.ismodule(parent): + faked = fake.get_faked(module and module.obj, obj) + if faked is not None: + faked.parent = parent + return faked + + return PyObject(obj, parent) diff --git a/jedi/evaluate/compiled/fake.py b/jedi/evaluate/compiled/fake.py new file mode 100644 index 00000000..345efe60 --- /dev/null +++ b/jedi/evaluate/compiled/fake.py @@ -0,0 +1,107 @@ +""" +Loads functions that are mixed in to the standard library. E.g. builtins are +written in C (binaries), but my autocompletion only understands Python code. By +mixing in Python code, the autocompletion should work much better for builtins. +""" + +import os +import inspect + +from jedi._compatibility import is_py3k, builtins +from jedi.parser import Parser +from jedi.parser.representation import Class +from jedi.evaluate.helpers import FakeName + +modules = {} + + +def _load_faked_module(module): + module_name = module.__name__ + if module_name == '__builtin__' and not is_py3k: + module_name = 'builtins' + + try: + return modules[module_name] + except KeyError: + path = os.path.dirname(os.path.abspath(__file__)) + try: + with open(os.path.join(path, 'fake', module_name) + '.pym') as f: + source = f.read() + except IOError: + modules[module_name] = None + return + module = Parser(source, module_name).module + modules[module_name] = module + + if module_name == 'builtins' and not is_py3k: + # There are two implementations of `open` for either python 2/3. + # -> Rename the python2 version (`look at fake/builtins.pym`). + open_func = search_scope(module, 'open') + open_func.name = FakeName('open_python3') + open_func = search_scope(module, 'open_python2') + open_func.name = FakeName('open') + return module + + +def search_scope(scope, obj_name): + for s in scope.subscopes: + if str(s.name) == obj_name: + return s + + +def _faked(module, obj, name): + # Crazy underscore actions to try to escape all the internal madness. + if module is None: + try: + module = obj.__objclass__ + except AttributeError: + pass + + try: + imp_plz = obj.__module__ + except AttributeError: + # Unfortunately in some cases like `int` there's no __module__ + module = builtins + else: + module = __import__(imp_plz) + + faked_mod = _load_faked_module(module) + if faked_mod is None: + return + + # Having the module as a `parser.representation.module`, we need to scan + # for methods. + if name is None: + if inspect.isbuiltin(obj): + return search_scope(faked_mod, obj.__name__) + elif not inspect.isclass(obj): + # object is a method or descriptor + cls = search_scope(faked_mod, obj.__objclass__.__name__) + if cls is None: + return + return search_scope(cls, obj.__name__) + else: + if obj == module: + return search_scope(faked_mod, name) + else: + cls = search_scope(faked_mod, obj.__name__) + if cls is None: + return + return search_scope(cls, name) + + +def get_faked(module, obj, name=None): + obj = obj.__class__ if is_class_instance(obj) else obj + result = _faked(module, obj, name) + if not isinstance(result, Class) and result is not None: + # Set the docstr which was previously not set (faked modules don't + # contain it). + result.docstr = obj.__doc__ or '' + return result + + +def is_class_instance(obj): + """Like inspect.* methods.""" + return not (inspect.isclass(obj) or inspect.ismodule(obj) + or inspect.isbuiltin(obj) or inspect.ismethod(obj) + or inspect.ismethoddescriptor(obj)) diff --git a/jedi/evaluate/mixin/_functools.pym b/jedi/evaluate/compiled/fake/_functools.pym similarity index 100% rename from jedi/evaluate/mixin/_functools.pym rename to jedi/evaluate/compiled/fake/_functools.pym diff --git a/jedi/evaluate/mixin/_sqlite3.pym b/jedi/evaluate/compiled/fake/_sqlite3.pym similarity index 100% rename from jedi/evaluate/mixin/_sqlite3.pym rename to jedi/evaluate/compiled/fake/_sqlite3.pym diff --git a/jedi/evaluate/mixin/_sre.pym b/jedi/evaluate/compiled/fake/_sre.pym similarity index 100% rename from jedi/evaluate/mixin/_sre.pym rename to jedi/evaluate/compiled/fake/_sre.pym diff --git a/jedi/evaluate/mixin/_weakref.pym b/jedi/evaluate/compiled/fake/_weakref.pym similarity index 91% rename from jedi/evaluate/mixin/_weakref.pym rename to jedi/evaluate/compiled/fake/_weakref.pym index 05eab2c8..8d21a2c4 100644 --- a/jedi/evaluate/mixin/_weakref.pym +++ b/jedi/evaluate/compiled/fake/_weakref.pym @@ -1,7 +1,7 @@ def proxy(object, callback=None): return object -class ref(): +class weakref(): def __init__(self, object, callback=None): self.__object = object def __call__(self): diff --git a/jedi/evaluate/mixin/builtins.pym b/jedi/evaluate/compiled/fake/builtins.pym similarity index 93% rename from jedi/evaluate/mixin/builtins.pym rename to jedi/evaluate/compiled/fake/builtins.pym index 19004012..2ab23b62 100644 --- a/jedi/evaluate/mixin/builtins.pym +++ b/jedi/evaluate/compiled/fake/builtins.pym @@ -45,6 +45,15 @@ class xrange(): return 1 +def open(file, mode='r', buffering=-1, encoding=None, errors=None, newline=None, closefd=True): + import io + return io.TextIOWrapper(file, mode, buffering, encoding, errors, newline, closefd) + + +def open_python2(name, mode=None, buffering=None): + return file(name, mode, buffering) + + #-------------------------------------------------------- # descriptors #-------------------------------------------------------- diff --git a/jedi/evaluate/mixin/datetime.pym b/jedi/evaluate/compiled/fake/datetime.pym similarity index 100% rename from jedi/evaluate/mixin/datetime.pym rename to jedi/evaluate/compiled/fake/datetime.pym diff --git a/jedi/evaluate/mixin/_io.pym b/jedi/evaluate/compiled/fake/io.pym similarity index 58% rename from jedi/evaluate/mixin/_io.pym rename to jedi/evaluate/compiled/fake/io.pym index 0feec3a2..8225477a 100644 --- a/jedi/evaluate/mixin/_io.pym +++ b/jedi/evaluate/compiled/fake/io.pym @@ -1,3 +1,3 @@ class TextIOWrapper(): def __next__(self): - return '' + return 'hacked io return' diff --git a/jedi/evaluate/mixin/posix.pym b/jedi/evaluate/compiled/fake/posix.pym similarity index 100% rename from jedi/evaluate/mixin/posix.pym rename to jedi/evaluate/compiled/fake/posix.pym diff --git a/jedi/evaluate/finder.py b/jedi/evaluate/finder.py index a03ec72c..007e202c 100644 --- a/jedi/evaluate/finder.py +++ b/jedi/evaluate/finder.py @@ -7,7 +7,7 @@ from jedi import common from jedi import settings from jedi.evaluate import representation as er from jedi.evaluate import dynamic -from jedi.evaluate import builtin +from jedi.evaluate import compiled from jedi.evaluate import docstrings from jedi.evaluate import iterable @@ -71,9 +71,10 @@ class NameFinder(object): if not result and isinstance(self.scope, er.Instance): # __getattr__ / __getattribute__ for r in self._check_getattr(self.scope): - new_name = copy.copy(r.name) - new_name.parent = r - result.append(new_name) + if not isinstance(r, compiled.PyObject): + new_name = copy.copy(r.name) + new_name.parent = r + result.append(new_name) debug.dbg('sfn filter "%s" in (%s-%s): %s@%s' % (self.name_str, self.scope, nscope, u(result), self.position)) @@ -82,9 +83,8 @@ class NameFinder(object): def _check_getattr(self, inst): """Checks for both __getattr__ and __getattribute__ methods""" result = [] - module = builtin.Builtin.scope # str is important to lose the NamePart! - name = pr.String(module, "'%s'" % self.name_str, (0, 0), (0, 0), inst) + name = compiled.create(str(self.name_str)) with common.ignored(KeyError): result = inst.execute_subscope_by_name('__getattr__', [name]) if not result: @@ -242,7 +242,8 @@ class NameFinder(object): c = r.expression_list()[0] if c in ('*', '**'): t = 'tuple' if c == '*' else 'dict' - res_new = evaluator.execute(evaluator.find_types(builtin.Builtin.scope, t)[0]) + typ = evaluator.find_types(compiled.builtin, t)[0] + res_new = evaluator.execute(typ) if not r.assignment_details: # this means that there are no default params, # so just ignore it. diff --git a/jedi/evaluate/helpers.py b/jedi/evaluate/helpers.py index cf2cf916..b298d8bb 100644 --- a/jedi/evaluate/helpers.py +++ b/jedi/evaluate/helpers.py @@ -156,3 +156,20 @@ def scan_statement_for_calls(stmt, search_name, assignment_details=False): s_new = s_new.next return result + + +class FakeSubModule(): + line_offset = 0 + + +class FakeStatement(pr.Statement): + def __init__(self, expression_list): + p = 0, 0 + super(FakeStatement, self).__init__(FakeSubModule, expression_list, p, p) + self._expression_list = expression_list + + +class FakeName(pr.Name): + def __init__(self, name, parent=None): + p = 0, 0 + super(FakeName, self).__init__(FakeSubModule, [(name, p)], p, p, parent) diff --git a/jedi/evaluate/imports.py b/jedi/evaluate/imports.py index 7176609d..61fec297 100644 --- a/jedi/evaluate/imports.py +++ b/jedi/evaluate/imports.py @@ -25,6 +25,7 @@ from jedi.parser import representation as pr from jedi.evaluate import sys_path from jedi import settings from jedi.common import source_to_unicode +from jedi.evaluate import compiled class ModuleNotFound(Exception): @@ -381,8 +382,9 @@ def strip_imports(evaluator, scopes): @cache.cache_star_import def remove_star_imports(evaluator, scope, ignored_modules=()): """ - Check a module for star imports: - >>> from module import * + Check a module for star imports:: + + from module import * and follow these modules. """ @@ -404,9 +406,7 @@ def load_module(path=None, source=None, name=None): with open(path) as f: source = f.read() else: - # TODO refactoring remove - from jedi.evaluate import builtin - return builtin.BuiltinModule(path, name).parser.module + return compiled.load_module(path, name) p = path or name p = fast.FastParser(common.source_to_unicode(source), p) cache.save_parser(path, name, p) @@ -436,7 +436,7 @@ def get_modules_containing_name(mods, name): return load_module(path, source) # skip non python modules - mods = set(m for m in mods if m.path is None or m.path.endswith('.py')) + mods = set(m for m in mods if not isinstance(m, compiled.PyObject)) mod_paths = set() for m in mods: mod_paths.add(m.path) diff --git a/jedi/evaluate/iterable.py b/jedi/evaluate/iterable.py index ba237fa1..63aeb801 100644 --- a/jedi/evaluate/iterable.py +++ b/jedi/evaluate/iterable.py @@ -3,9 +3,9 @@ import itertools from jedi import common from jedi import debug from jedi import settings -from jedi._compatibility import use_metaclass, is_py3k +from jedi._compatibility import use_metaclass, is_py3k, unicode from jedi.parser import representation as pr -from jedi.evaluate import builtin +from jedi.evaluate import compiled from jedi.evaluate import helpers from jedi.evaluate.cache import CachedMetaClass, memoize_default @@ -24,16 +24,10 @@ class Generator(use_metaclass(CachedMetaClass, pr.Base)): content of a generator. """ names = [] - none_pos = (0, 0) executes_generator = ('__next__', 'send') for n in ('close', 'throw') + executes_generator: - name = pr.Name(builtin.Builtin.scope, [(n, none_pos)], - none_pos, none_pos) - if n in executes_generator: - name.parent = self - else: - name.parent = builtin.Builtin.scope - names.append(name) + parent = self if n in executes_generator else compiled.builtin + names.append(helpers.FakeName(n, parent)) debug.dbg('generator names', names) return names @@ -78,16 +72,10 @@ class Array(use_metaclass(CachedMetaClass, pr.Base)): # This is indexing only one element, with a fixed index number, # otherwise it just ignores the index (e.g. [1+1]). index = index_possibilities[0] - - from jedi.evaluate.representation import Instance - if isinstance(index, Instance) \ - and str(index.name) in ['int', 'str'] \ - and len(index.var_args) == 1: - # TODO this is just very hackish and a lot of use cases are - # being ignored - with common.ignored(KeyError, IndexError, - UnboundLocalError, TypeError): - return self.get_exact_index_types(index.var_args[0]) + if isinstance(index, compiled.PyObject) \ + and isinstance(index.obj, (int, str, unicode)): + with common.ignored(KeyError, IndexError, TypeError): + return self.get_exact_index_types(index.obj) result = list(self._follow_values(self._array.values)) result += check_array_additions(self._evaluator, self) @@ -108,6 +96,8 @@ class Array(use_metaclass(CachedMetaClass, pr.Base)): str_key = key.value elif isinstance(key, pr.Name): str_key = str(key) + else: + continue if mixed_index == str_key: index = i @@ -130,17 +120,17 @@ class Array(use_metaclass(CachedMetaClass, pr.Base)): It returns e.g. for a list: append, pop, ... """ # `array.type` is a string with the type, e.g. 'list'. - scope = self._evaluator.find_types(builtin.Builtin.scope, self._array.type)[0] + scope = self._evaluator.find_types(compiled.builtin, self._array.type)[0] scope = self._evaluator.execute(scope)[0] # builtins only have one class names = scope.get_defined_names() return [ArrayMethod(n) for n in names] @property def parent(self): - return builtin.Builtin.scope + return compiled.builtin def get_parent_until(self): - return builtin.Builtin.scope + return compiled.builtin def __getattr__(self, name): if name not in ['type', 'start_pos', 'get_only_subelement', 'parent', @@ -177,7 +167,7 @@ class ArrayMethod(object): return getattr(self.name, name) def get_parent_until(self): - return builtin.Builtin.scope + return compiled.builtin def __repr__(self): return "<%s of %s>" % (type(self).__name__, self.name) @@ -240,7 +230,7 @@ def _check_array_additions(evaluator, compare_array, module, is_list): >>> a = [""] >>> a.append(1) """ - if not settings.dynamic_array_additions or module.is_builtin(): + if not settings.dynamic_array_additions or isinstance(module, compiled.PyObject): return [] def check_calls(calls, add_name): diff --git a/jedi/evaluate/param.py b/jedi/evaluate/param.py index 449aa513..e6c2ee45 100644 --- a/jedi/evaluate/param.py +++ b/jedi/evaluate/param.py @@ -2,8 +2,8 @@ import copy from jedi.parser import representation as pr from jedi.evaluate import iterable -from jedi.evaluate import builtin from jedi.evaluate import common +from jedi.evaluate import helpers def get_params(evaluator, func, var_args): @@ -24,11 +24,11 @@ def get_params(evaluator, func, var_args): new_param.parent = parent # create an Array (-> needed for *args/**kwargs tuples/dicts) - arr = pr.Array(_FakeSubModule, start_pos, array_type, parent) + arr = pr.Array(helpers.FakeSubModule, start_pos, array_type, parent) arr.values = values key_stmts = [] for key in keys: - stmt = pr.Statement(_FakeSubModule, [], start_pos, None) + stmt = pr.Statement(helpers.FakeSubModule, [], start_pos, None) stmt._expression_list = [key] key_stmts.append(stmt) arr.keys = key_stmts @@ -140,8 +140,7 @@ def _var_args_iterator(evaluator, var_args): continue old = stmt # generate a statement if it's not already one. - module = builtin.Builtin.scope - stmt = pr.Statement(module, [], (0, 0), None) + stmt = pr.Statement(helpers.FakeSubModule, [], (0, 0), None) stmt._expression_list = [old] # *args @@ -156,7 +155,7 @@ def _var_args_iterator(evaluator, var_args): yield None, field_stmt elif isinstance(array, iterable.Generator): for field_stmt in array.iter_content(): - yield None, _FakeStatement(field_stmt) + yield None, helpers.FakeStatement([field_stmt]) # **kwargs elif expression_list[0] == '**': for array in evaluator.eval_expression_list(expression_list[1:]): @@ -177,13 +176,3 @@ def _var_args_iterator(evaluator, var_args): yield key_arr[0].name, stmt else: yield None, stmt - - -class _FakeSubModule(): - line_offset = 0 - - -class _FakeStatement(pr.Statement): - def __init__(self, content): - p = 0, 0 - super(_FakeStatement, self).__init__(_FakeSubModule, [content], p, p) diff --git a/jedi/evaluate/recursion.py b/jedi/evaluate/recursion.py index 51bb598e..620ed93e 100644 --- a/jedi/evaluate/recursion.py +++ b/jedi/evaluate/recursion.py @@ -10,7 +10,7 @@ calls. from jedi.parser import representation as pr from jedi import debug from jedi import settings -from jedi.evaluate import builtin +from jedi.evaluate import compiled from jedi.evaluate import iterable @@ -82,7 +82,7 @@ class _RecursionNode(object): # The same's true for the builtins, because the builtins are really # simple. self.is_ignored = isinstance(stmt, pr.Param) \ - or (self.script == builtin.Builtin.scope) + or (self.script == compiled.builtin) def __eq__(self, other): if not other: @@ -148,7 +148,7 @@ class ExecutionRecursionDetector(object): if isinstance(execution.base, (iterable.Array, iterable.Generator)): return False module = execution.get_parent_until() - if evaluate_generator or module == builtin.Builtin.scope: + if evaluate_generator or module == compiled.builtin: return False if in_par_execution_funcs: diff --git a/jedi/evaluate/representation.py b/jedi/evaluate/representation.py index f9ea1f37..df9ff143 100644 --- a/jedi/evaluate/representation.py +++ b/jedi/evaluate/representation.py @@ -16,7 +16,7 @@ from jedi.parser import representation as pr from jedi import debug from jedi import common from jedi.evaluate.cache import memoize_default, CachedMetaClass -from jedi.evaluate import builtin +from jedi.evaluate import compiled from jedi.evaluate import recursion from jedi.evaluate import iterable from jedi.evaluate import docstrings @@ -49,7 +49,7 @@ class Instance(use_metaclass(CachedMetaClass, Executable)): def __init__(self, evaluator, base, var_args=()): super(Instance, self).__init__(evaluator, base, var_args) if str(base.name) in ['list', 'set'] \ - and builtin.Builtin.scope == base.get_parent_until(): + and compiled.builtin == base.get_parent_until(): # compare the module path with the builtin name. self.var_args = iterable.check_array_instances(evaluator, self) else: @@ -113,9 +113,10 @@ class Instance(use_metaclass(CachedMetaClass, Executable)): if n.names[0] == self_name and len(n.names) == 2: add_self_dot_name(n) - for s in self.base.get_super_classes(): - for inst in self._evaluator.execute(s): - names += inst.get_self_attributes() + if not isinstance(self.base, compiled.PyObject): + for s in self.base.get_super_classes(): + for inst in self._evaluator.execute(s): + names += inst.get_self_attributes() return names def get_subscope_by_name(self, name): @@ -141,8 +142,7 @@ class Instance(use_metaclass(CachedMetaClass, Executable)): """ names = self.get_self_attributes() - class_names = self.base.instance_names() - for var in class_names: + for var in self.base.instance_names(): names.append(InstanceElement(self._evaluator, self, var, True)) return names @@ -154,8 +154,7 @@ class Instance(use_metaclass(CachedMetaClass, Executable)): yield self, self.get_self_attributes() names = [] - class_names = self.base.instance_names() - for var in class_names: + for var in self.base.instance_names(): names.append(InstanceElement(self._evaluator, self, var, True)) yield self, names @@ -199,10 +198,10 @@ class InstanceElement(use_metaclass(CachedMetaClass, pr.Base)): def parent(self): par = self.var.parent if isinstance(par, Class) and par == self.instance.base \ - or isinstance(par, pr.Class) \ + or isinstance(par, pr.Class) \ and par == self.instance.base.base: par = self.instance - elif not isinstance(par, pr.Module): + elif not isinstance(par, (pr.Module, compiled.PyObject)): par = InstanceElement(self.instance._evaluator, self.instance, par, self.is_class_var) return par @@ -256,13 +255,14 @@ class Class(use_metaclass(CachedMetaClass, pr.IsScope)): debug.warning('Received non class, as a super class') continue # Just ignore other stuff (user input error). supers.append(cls) - if not supers and self.base.parent != builtin.Builtin.scope: + if not supers and self.base.parent != compiled.builtin: # add `object` to classes - supers += self._evaluator.find_types(builtin.Builtin.scope, 'object') + supers += self._evaluator.find_types(compiled.builtin, 'object') return supers @memoize_default(default=()) def instance_names(self): + # TODO REMOVE instance_names def in_iterable(name, iterable): """ checks if the name is in the variable 'iterable'. """ for i in iterable: @@ -277,17 +277,20 @@ class Class(use_metaclass(CachedMetaClass, pr.IsScope)): # TODO mro! for cls in self.get_super_classes(): # Get the inherited names. - for i in cls.instance_names(): - if not in_iterable(i, result): - super_result.append(i) + if isinstance(cls, compiled.PyObject): + super_result += cls.get_defined_names() + else: + for i in cls.instance_names(): + if not in_iterable(i, result): + super_result.append(i) result += super_result return result @memoize_default(default=()) def get_defined_names(self): result = self.instance_names() - type_cls = self._evaluator.find_types(builtin.Builtin.scope, 'type')[0] - return result + type_cls.base.get_defined_names() + type_cls = self._evaluator.find_types(compiled.builtin, 'type')[0] + return result + list(type_cls.get_defined_names()) def get_subscope_by_name(self, name): for sub in reversed(self.subscopes): @@ -373,11 +376,11 @@ class Function(use_metaclass(CachedMetaClass, pr.IsScope)): return Function(self._evaluator, self.base_func, True) return decorated_func - def get_magic_method_names(self): - return builtin.Builtin.magic_function_scope(self._evaluator).get_defined_names() + def get_magic_function_names(self): + return compiled.magic_function_class.get_defined_names() - def get_magic_method_scope(self): - return builtin.Builtin.magic_function_scope(self._evaluator) + def get_magic_function_scope(self): + return compiled.magic_function_class def __getattr__(self, name): return getattr(self.base_func, name) diff --git a/jedi/evaluate/stdlib.py b/jedi/evaluate/stdlib.py index 25857d9f..3d31f30d 100644 --- a/jedi/evaluate/stdlib.py +++ b/jedi/evaluate/stdlib.py @@ -3,7 +3,8 @@ Implementations of standard library functions, because it's not possible to understand them with Jedi. """ -from jedi.evaluate import builtin +from jedi._compatibility import unicode +from jedi.evaluate import compiled from jedi.evaluate import representation as er from jedi.evaluate import iterable from jedi.parser import representation as pr @@ -17,7 +18,7 @@ class NotInStdLib(LookupError): def execute(evaluator, obj, params): if not isinstance(obj, (iterable.Generator, iterable.Array)): obj_name = str(obj.name) - if obj.parent == builtin.Builtin.scope: + if obj.parent == compiled.builtin: # for now we just support builtin functions. try: return _implemented['builtins'][obj_name](evaluator, obj, params) @@ -44,18 +45,17 @@ def builtins_getattr(evaluator, obj, params): objects = _follow_param(evaluator, params, 0) names = _follow_param(evaluator, params, 1) for obj in objects: - if not isinstance(obj, (er.Instance, er.Class, pr.Module)): + if not isinstance(obj, (er.Instance, er.Class, pr.Module, compiled.PyObject)): debug.warning('getattr called without instance') continue - for arr_name in names: - if not isinstance(arr_name, er.Instance): + for name in names: + s = unicode, str + if isinstance(name, compiled.PyObject) and isinstance(name.obj, s): + stmts += evaluator.follow_path(iter([name.obj]), [obj], obj) + else: debug.warning('getattr called without str') continue - if len(arr_name.var_args) != 1: - debug.warning('jedi getattr is too simple') - key = arr_name.var_args[0] - stmts += evaluator.follow_path(iter([key]), [obj], obj) return stmts diff --git a/jedi/keywords.py b/jedi/keywords.py index ab0fd4b3..7fea1b31 100644 --- a/jedi/keywords.py +++ b/jedi/keywords.py @@ -2,9 +2,8 @@ import pydoc import keyword from jedi._compatibility import is_py3k -from jedi.parser import representation as pr from jedi import common -from jedi.evaluate import builtin +from jedi.evaluate import compiled try: from pydoc_data import topics as pydoc_topics @@ -30,8 +29,7 @@ def keyword_names(*args, **kwargs): kwds = [] for k in keywords(*args, **kwargs): start = k.start_pos - end = start[0], start[1] + len(k.name) - kwds.append(pr.Name(k.parent, [(k.name, start)], start, end, k)) + kwds.append(KeywordName(k, k.name, start)) return kwds @@ -39,11 +37,22 @@ def get_operator(string, pos): return Keyword(string, pos) +class KeywordName(object): + def __init__(self, parent, name, start_pos): + self.parent = parent + self.names = [name] + self.start_pos = start_pos + + @property + def end_pos(self): + return self.start_pos[0], self.start_pos[1] + len(self.name) + + class Keyword(object): def __init__(self, name, pos): self.name = name self.start_pos = pos - self.parent = builtin.Builtin.scope + self.parent = compiled.builtin def get_parent_until(self): return self.parent diff --git a/jedi/parser/representation.py b/jedi/parser/representation.py index b9feea5e..60973ade 100644 --- a/jedi/parser/representation.py +++ b/jedi/parser/representation.py @@ -362,17 +362,13 @@ class SubModule(Scope, Module): string = '' # no path -> empty name else: sep = (re.escape(os.path.sep),) * 2 - r = re.search(r'([^%s]*?)(%s__init__)?(\.py|\.so)?$' % sep, - self.path) + r = re.search(r'([^%s]*?)(%s__init__)?(\.py|\.so)?$' % sep, self.path) # remove PEP 3149 names string = re.sub('\.[a-z]+-\d{2}[mud]{0,3}$', '', r.group(1)) # positions are not real therefore choose (0, 0) names = [(string, (0, 0))] return Name(self, names, (0, 0), (0, 0), self.use_as_parent) - def is_builtin(self): - return not (self.path is None or self.path.endswith('.py')) - @property def has_explicit_absolute_import(self): """ @@ -1239,9 +1235,6 @@ class Literal(StatementElement): def get_code(self): return self.literal + super(Literal, self).get_code() - def type_as_string(self): - return type(self.value).__name__ - def __repr__(self): if is_py3k: s = self.literal diff --git a/pytest.ini b/pytest.ini index 21f00ce2..fa132557 100644 --- a/pytest.ini +++ b/pytest.ini @@ -2,7 +2,7 @@ addopts = --doctest-modules # Ignore broken files in blackbox test directories -norecursedirs = .* docs completion refactor absolute_import namespace_package +norecursedirs = .* docs completion refactor absolute_import namespace_package scripts # Activate `clean_jedi_cache` fixture for all tests. This should be # fine as long as we are using `clean_jedi_cache` as a session scoped diff --git a/scripts/memory_check.py b/scripts/memory_check.py new file mode 100644 index 00000000..e77c9dfd --- /dev/null +++ b/scripts/memory_check.py @@ -0,0 +1,53 @@ +""" +This is a convenience script to test the speed and memory usage of Jedi with +large libraries. + +Each library is preloaded by jedi, recording the time and memory consumed by +each operation. + +You can provide additional libraries via command line arguments. + +Note: This requires the psutil library, available on PyPI. +""" +import time +import sys +import psutil +import jedi + + +def used_memory(): + """Return the total MB of System Memory in use.""" + return psutil.virtual_memory().used / 2**20 + + +def profile_preload(mod): + """Preload a module into Jedi, recording time and memory used.""" + base = used_memory() + t0 = time.time() + jedi.preload_module(mod) + elapsed = time.time() - t0 + used = used_memory() - base + return elapsed, used + + +def main(mods): + """Preload the modules, and print the time and memory used.""" + t0 = time.time() + baseline = used_memory() + print('Time (s) | Mem (MB) | Package') + print('------------------------------') + for mod in mods: + elapsed, used = profile_preload(mod) + if used > 0: + print('%8.1f | %8d | %s' % (elapsed, used, mod)) + print('------------------------------') + elapsed = time.time() - t0 + used = used_memory() - baseline + print('%8.1f | %8d | %s' % (elapsed, used, 'Total')) + + +if __name__ == '__main__': + mods = ['re', 'numpy', 'scipy', 'scipy.sparse', 'scipy.stats', + 'wx', 'decimal', 'PyQt4.QtGui', 'PySide.QtGui', 'Tkinter'] + mods += sys.argv[1:] + main(mods) diff --git a/setup.py b/setup.py index 2fb4f277..6b6f8963 100755 --- a/setup.py +++ b/setup.py @@ -26,8 +26,8 @@ setup(name='jedi', license='MIT', keywords='python completion refactoring vim', long_description=readme, - packages=['jedi', 'jedi.parser', 'jedi.evaluate', 'jedi.api'], - package_data={'jedi': ['evlaluate/evaluate/mixin/*.pym']}, + packages=['jedi', 'jedi.parser', 'jedi.evaluate', 'jedi.evaluate.compiled', 'jedi.api'], + package_data={'jedi': ['evlaluate/evaluate/compiled/fake/*.pym']}, platforms=['any'], classifiers=[ 'Development Status :: 4 - Beta', diff --git a/test/test_api.py b/test/test_api.py index 66e2c5ff..44f94c7b 100644 --- a/test/test_api.py +++ b/test/test_api.py @@ -2,27 +2,25 @@ Test all things related to the ``jedi.api`` module. """ -from jedi import common, api +from jedi import api from pytest import raises def test_preload_modules(): def check_loaded(*modules): - # + 1 for builtin, +1 for None module (currently used) - assert len(new) == len(modules) + 2 - for i in modules + ('__builtin__',): - assert [i in k for k in new.keys() if k is not None] + # +1 for None module (currently used) + assert len(parser_cache) == len(modules) + 1 + for i in modules: + assert [i in k for k in parser_cache.keys() if k is not None] from jedi import cache temp_cache, cache.parser_cache = cache.parser_cache, {} - new = cache.parser_cache - with common.ignored(KeyError): # performance of tests -> no reload - new['__builtin__'] = temp_cache['__builtin__'] + parser_cache = cache.parser_cache - api.preload_module('datetime') - check_loaded('datetime') + api.preload_module('sys') + check_loaded() # compiled (c_builtin) modules shouldn't be in the cache. api.preload_module('json', 'token') - check_loaded('datetime', 'json', 'token') + check_loaded('json', 'token') cache.parser_cache = temp_cache @@ -30,6 +28,7 @@ def test_preload_modules(): def test_empty_script(): assert api.Script('') + def test_line_number_errors(): """ Script should raise a ValueError if line/column numbers are not in a diff --git a/test/test_api_classes.py b/test/test_api_classes.py index 6933a1a8..43a99db7 100644 --- a/test/test_api_classes.py +++ b/test/test_api_classes.py @@ -15,6 +15,7 @@ def test_is_keyword(): results = Script('str', 1, 1, None).goto_definitions() assert len(results) == 1 and results[0].is_keyword is False + def make_definitions(): """ Return a list of definitions for parametrized tests. diff --git a/test/test_builtin.py b/test/test_builtin.py deleted file mode 100644 index b9513c30..00000000 --- a/test/test_builtin.py +++ /dev/null @@ -1,13 +0,0 @@ -from jedi.evaluate import builtin - - -def test_parse_function_doc_illegal_docstr(): - - def test_func(a): - """ - test_func(o - - doesn't have a closing bracket. - """ - - assert ('', '') == builtin._parse_function_doc(test_func) diff --git a/test/test_call_signatures.py b/test/test_call_signatures.py index 9505b3ae..6fe99a9e 100644 --- a/test/test_call_signatures.py +++ b/test/test_call_signatures.py @@ -15,7 +15,7 @@ class TestCallSignatures(TestCase): assert signatures[0].call_name == expected_name assert signatures[0].index == expected_index - def test_call_signatures(self): + def test_simple(self): def run(source, name, index=0, column=None, line=1): self._run(source, name, index, line, column) @@ -72,7 +72,7 @@ class TestCallSignatures(TestCase): "func(alpha='101'," run(s, 'func', 0, column=13, line=2) - def test_function_definition_complex(self): + def test_complex(self): s = """ def abc(a,b): pass @@ -106,7 +106,7 @@ class TestCallSignatures(TestCase): # just don't throw an exception (if numpy doesn't exist, just ignore it) assert Script(s).call_signatures() == [] - def test_function_definition_empty_paren_pre_space(self): + def test_call_signatures_empty_parentheses_pre_space(self): s = textwrap.dedent("""\ def f(a, b): pass diff --git a/test/test_compiled.py b/test/test_compiled.py new file mode 100644 index 00000000..c358fef6 --- /dev/null +++ b/test/test_compiled.py @@ -0,0 +1,36 @@ +from jedi._compatibility import builtins +from jedi.parser.representation import Function +from jedi.evaluate import compiled +from jedi.evaluate import Evaluator + + +def test_simple(): + e = Evaluator() + bltn = compiled.PyObject(builtins) + obj = compiled.PyObject('_str_', bltn) + upper = e.find_types(obj, 'upper') + assert len(upper) == 1 + objs = list(e.execute(upper[0])) + assert len(objs) == 1 + assert objs[0].obj is str + + +def test_fake_loading(): + assert isinstance(compiled.create(next), Function) + + string = compiled.builtin.get_subscope_by_name('str') + from_name = compiled._create_from_name(compiled.builtin, string, '__init__') + assert isinstance(from_name, Function) + + +def test_fake_docstr(): + assert compiled.create(next).docstr == next.__doc__ + + +def test_parse_function_doc_illegal_docstr(): + docstr = """ + test_func(o + + doesn't have a closing bracket. + """ + assert ('', '') == compiled._parse_function_doc(docstr)