diff --git a/builtin.py b/builtin.py new file mode 100644 index 00000000..d8ae01af --- /dev/null +++ b/builtin.py @@ -0,0 +1,161 @@ +import re + +import debug +import parsing + + +class Parser(object): + map_types = { + 'floating point number': '0.0', + 'string': '""', + 'str': '""', + 'integer': '0', + 'int': '0', + 'dictionary': '{}', + 'list': '[]', + 'object': '{}', + # TODO things like dbg: ('not working', 'tuple of integers') + } + + """ This module tries to imitate parsing.Scope """ + def __init__(self, name): + self.name = name + self.parent = None + self.content = {} + exec 'import %s as module' % name in self.content + self.module = self.content['module'] + self._parser = None + + @property + def parser(self): + """ get the parser lazy """ + if self._parser: + return self._parser + else: + code = self.generate_code(self.module) + try: + self._parser = parsing.PyFuzzyParser(code) + except: + debug.warning('not possible to resolve', self.name, code) + #open('builtin_fail', 'w').write(code) + raise + return self._parser + + def generate_code(self, scope, depth=0): + """ + Generate a string, which uses python syntax as an input to the + PyFuzzyParser. + """ + def get_types(names): + classes = {} + funcs = {} + stmts = {} + members = {} + for n in names: + if '__' in n: + continue + # this has a builtin_function_or_method + exe = getattr(scope, n) + if type(exe).__name__ == 'builtin_function_or_method': + funcs[n] = exe + elif type(exe) == type: + classes[n] = exe + elif type(exe).__name__ == 'member_descriptor': + members[n] = exe + else: + stmts[n] = exe + return classes, funcs, stmts, members + + code = '' + try: + try: + path = scope.__file__ + except: + path = '?' + code += '# Generated module %s from %s\n' % (scope.__name__, path) + except: + pass + code += '"""\n%s\n"""\n' % scope.__doc__ + + names = set(dir(scope)) - {'__file__', '__name__', '__doc__', + '__path__', '__package__'} + classes, funcs, stmts, members = get_types(names) + #print 'blub', get_types(names) + + # classes + for name, cl in classes.iteritems(): + bases = (c.__name__ for c in cl.__bases__) + code += 'class %s(%s):\n' % (name, ','.join(bases)) + if depth == 0: + cl_code = self.generate_code(cl, depth + 1) + code += parsing.indent_block(cl_code) + code += '\n' + + # functions + for name, func in funcs.iteritems(): + params, ret = parse_function_doc(func) + code += 'def %s(%s):\n' % (name, params) + block = '"""\n%s\n"""\n' % func.__doc__ + block += '%s\n\n' % ret + code += parsing.indent_block(block) + + # class members (functions) + for name, func in members.iteritems(): + ret = 'pass' + code += '@property\ndef %s(self):\n' % (name) + block = '"""\n%s\n"""\n' % func.__doc__ + block += '%s\n\n' % ret + code += parsing.indent_block(block) + + # variables + for name, value in stmts.iteritems(): + if isinstance(value, str): + value = repr(value) + elif type(value) == file: + value = 'file' + code += '%s = %s\n' % (name, value) + + import sys + #if depth == 0: + # sys.stdout.write(code) + # exit() + return code + + +def parse_function_doc(func): + """ + Takes a function and returns the params and return value as a tuple. + This is nothing more than a docstring parser. + """ + # TODO: things like utime(path, (atime, mtime)) and a(b [, b]) -> None + params = [] + doc = func.__doc__ + end = doc.index(')') + #print 'blubedi', doc + param_str = doc[doc.index('(') + 1:end] + try: + index = doc.index('-> ', end, end + 7) + except ValueError: + ret = 'pass' + else: + pattern = re.compile(r'(,\n|[^\n])+') + ret_str = pattern.match(doc, index + 3).group(0) + ret = Parser.map_types.get(ret_str, ret_str) + if ret == ret_str and ret not in ['None', 'object', 'tuple', 'set']: + debug.dbg('not working', ret_str) + if ret != 'pass': + ret = 'return ' + ret + return param_str, ret + +"""if current.arr_type == parsing.Array.EMPTY: + # the normal case - no array type + print 'length', len(current) +elif current.arr_type == parsing.Array.LIST: + result.append(__builtin__.list()) +elif current.arr_type == parsing.Array.SET: + result.append(__builtin__.set()) +elif current.arr_type == parsing.Array.TUPLE: + result.append(__builtin__.tuple()) +elif current.arr_type == parsing.Array.DICT: + result.append(__builtin__.dict()) + """ diff --git a/debug.py b/debug.py index 66899168..113037b8 100644 --- a/debug.py +++ b/debug.py @@ -1,15 +1,27 @@ import inspect +NOTICE = object() +WARNING = object() +ERROR = object() + def dbg(*args): if debug_function: frm = inspect.stack()[1] mod = inspect.getmodule(frm[0]) if not (mod.__name__ in ignored_modules): - debug_function(*args) + debug_function(NOTICE, *args) def warning(*args): if debug_function: - debug_function(*args) + debug_function(WARNING, *args) -debug_function = None +def error(*args): + if debug_function: + debug_function(ERROR, *args) + +def print_to_stdout(level, *args): + """ The default debug function """ + print 'dbg:' if level == NOTICE else 'warning:', args + +debug_function = print_to_stdout ignored_modules = [] diff --git a/evaluate.py b/evaluate.py index 6c0f5803..ab63b403 100644 --- a/evaluate.py +++ b/evaluate.py @@ -1,3 +1,9 @@ +""" +follow_statement -> follow_call -> follow_paths -> follow_path +'follow_import' + +`get_names_for_scope` and `get_scopes_for_name` are search functions +""" import itertools import parsing @@ -8,6 +14,7 @@ import debug class Exec(object): def __init__(self, base): self.base = base + def get_parent_until(self, *args): return self.base.get_parent_until(*args) @@ -46,49 +53,39 @@ class Execution(Exec): """ This class is used to evaluate functions and their returns. """ - cache = {} + def get_return_types(self): """ Get the return vars of a function. """ def remove_executions(scope, get_returns=False): - if isinstance(scope, Execution): + stmts = [] + if isinstance(scope, parsing.Class): # there maybe executions of executions - stmts = scope.get_return_types() + stmts = [Instance(scope)] else: if get_returns: - stmts = scope.returns + ret = scope.returns + for s in ret: + for stmt in follow_statement(s): + stmts += remove_executions(stmt) else: - stmts = [scope] + stmts.append(scope) return stmts # check cache try: + debug.dbg('hit function cache', self.base) return Execution.cache[self.base] except KeyError: # cache is not only here as a cache, but also to prevent an # endless recursion. Execution.cache[self.base] = [] - result = [] - stmts = remove_executions(self.base, True) - print 'stmts=', stmts, self.base, repr(self) + result = remove_executions(self.base, True) + debug.dbg('exec stmts=', result, self.base, repr(self)) - #n += self.function.get_set_vars() - # these are the statements of the return functions - for stmt in stmts: - if isinstance(stmt, parsing.Class): - # it might happen, that a function returns a Class and this - # gets executed, therefore get the instance here. - result.append(Instance(stmt)) - else: - print 'addstmt', stmt - for followed in follow_statement(stmt): - print 'followed', followed - result += remove_executions(followed) - - print 'ret', stmt Execution.cache[self.base] = result return result @@ -106,11 +103,10 @@ def get_names_for_scope(scope): if not isinstance(scope, parsing.Class) or scope == start_scope: compl += scope.get_set_vars() scope = scope.parent - print 'get_names_for_scope', scope, len(compl) return compl -def get_scopes_for_name(scope, name, search_global=False, search_func=None): +def get_scopes_for_name(scope, name, search_global=False): """ :return: List of Names. Their parents are the scopes, they are defined in. :rtype: list @@ -129,6 +125,7 @@ def get_scopes_for_name(scope, name, search_global=False, search_func=None): res_new += remove_statements(scopes) else: res_new.append(r) + debug.dbg('sfn remove', res_new, result) return res_new def filter_name(scopes): @@ -137,6 +134,7 @@ def get_scopes_for_name(scope, name, search_global=False, search_func=None): for scope in scopes: if isinstance(scope, parsing.Import): try: + debug.dbg('star import', scope) i = follow_import(scope).get_defined_names() except modules.ModuleNotFound: debug.dbg('StarImport not found: ' + str(scope)) @@ -145,20 +143,40 @@ def get_scopes_for_name(scope, name, search_global=False, search_func=None): else: if [name] == list(scope.names): result.append(scope.parent) + debug.dbg('sfn filter', result) return result - if search_func: - names = search_func() - elif search_global: + if search_global: names = get_names_for_scope(scope) else: names = scope.get_set_vars() - # TODO here are the star imports handled, we need to get the names here. - # This means things like from pylab import * return remove_statements(filter_name(names)) +def resolve_results(scopes): + """ Here we follow the results - to get what we really want """ + result = [] + for s in scopes: + if isinstance(s, parsing.Import): + print 'dini mueter, steile griech!' + try: + scope = follow_import(s) + #for r in resolve_results([follow_import(s)]): + # if isinstance(r, parsing.Import): + # resolve_results(r) + # else: + # resolve + except modules.ModuleNotFound: + debug.dbg('Module not found: ' + str(s)) + else: + result.append(scope) + result += resolve_results(i for i in scope.get_imports() if i.star) + else: + result.append(s) + return result + + def follow_statement(stmt, scope=None): """ :param stmt: contains a statement @@ -168,41 +186,28 @@ def follow_statement(stmt, scope=None): scope = stmt.get_parent_until(parsing.Function) result = [] calls = stmt.get_assignment_calls() - print 'calls', calls, calls.values + debug.dbg('calls', calls, calls.values) for tokens in calls: for tok in tokens: - print 'tok', tok, type(tok), isinstance(tok,str) if not isinstance(tok, str): # the string tokens are just operations (+, -, etc.) result += follow_call(scope, tok) return result + def follow_call(scope, call): + """ Follow a call is following a function, variable, string, etc. """ path = call.generate_call_list() current = next(path) - result = [] if isinstance(current, parsing.Array): - """if current.arr_type == parsing.Array.EMPTY: - # the normal case - no array type - print 'length', len(current) - elif current.arr_type == parsing.Array.LIST: - result.append(__builtin__.list()) - elif current.arr_type == parsing.Array.SET: - result.append(__builtin__.set()) - elif current.arr_type == parsing.Array.TUPLE: - result.append(__builtin__.tuple()) - elif current.arr_type == parsing.Array.DICT: - result.append(__builtin__.dict()) - """ - result.append(current) + result = [current] else: - result = get_scopes_for_name(scope, current, search_global=True) - pass + scopes = get_scopes_for_name(scope, current, search_global=True) + result = resolve_results(scopes) - print 'before', result + debug.dbg('call before', result, current, scope) result = follow_paths(path, result) - print 'after result', result return result @@ -210,12 +215,11 @@ def follow_call(scope, call): def follow_paths(path, results): results_new = [] try: - if len(results) > 1: - iter_paths = itertools.tee(path, len(results)) - else: - iter_paths = [path] - print 'enter', results, len(results) - if len(results): + if results: + if len(results) > 1: + iter_paths = itertools.tee(path, len(results)) + else: + iter_paths = [path] for i, r in enumerate(results): results_new += follow_path(iter_paths[i], r) except StopIteration: @@ -225,21 +229,11 @@ def follow_paths(path, results): def follow_path(path, input): """ - takes a generator and tries to complete the path + Takes a generator and tries to complete the path. """ - def add_results(scopes): - """ Here we follow the results - to get what we really want """ - result = [] - for s in scopes: - if isinstance(s, parsing.Import): - print 'dini mueter, steile griech!' - try: - result.append(follow_import(s)) - except modules.ModuleNotFound: - debug.dbg('Module not found: ' + str(s)) - else: - result.append(s) - return result + # current is either an Array or a Scope + current = next(path) + debug.dbg('follow', current, input) def filter_result(scope): result = [] @@ -247,30 +241,24 @@ def follow_path(path, input): # this must be an execution, either () or [] if current.arr_type == parsing.Array.LIST: result = [] # TODO eval lists - else: + elif current.arr_type not in [parsing.Array.DICT, parsing]: # scope must be a class or func - make an instance or execution - if isinstance(scope, parsing.Class): - result.append(Instance(scope)) - else: - #try: - print '\n\n\n\n\nbefexec', scope - stmts = add_results(Execution(scope).get_return_types()) - debug.dbg('exec', stmts) - result = stmts - #except AttributeError: - # debug.dbg('cannot execute:', scope) + debug.dbg('befexec', scope) + result = resolve_results(Execution(scope).get_return_types()) + debug.dbg('exec', result) + #except AttributeError: + # debug.dbg('cannot execute:', scope) + else: + # curly braces are not allowed, because they make no sense + debug.warning('strange function call with {}', current, scope) else: if isinstance(scope, parsing.Function): # TODO check default function methods and return them result = [] else: # TODO check magic class methods and return them also - result = add_results(get_scopes_for_name(scope, current)) + result = resolve_results(get_scopes_for_name(scope, current)) return result - - current = next(path) - print 'follow', input, current - return follow_paths(path, filter_result(input)) @@ -293,4 +281,3 @@ def follow_import(_import): debug.dbg('after import', scope, rest) return scope - diff --git a/ftest.py b/ftest.py new file mode 100755 index 00000000..6486fde4 --- /dev/null +++ b/ftest.py @@ -0,0 +1,13 @@ +#!/usr/bin/env python + +import functions + +#functions.debug.debug_function +functions.debug.ignored_modules += ['parsing', 'functions'] +functions.modules.module_find_path.insert(0, '.') + +with open('test.py') as f: + code = f.read() +completions = functions.complete(code, 50, 20) + +print '\n', [c.names for c in completions] diff --git a/functions.py b/functions.py index da79124d..d6ac71d0 100644 --- a/functions.py +++ b/functions.py @@ -171,8 +171,8 @@ def complete(source, row, column, file_callback=None): debug.dbg('-' * 70) debug.dbg(' ' * 62 + 'complete') debug.dbg('-' * 70) - print scope - print f.parser.user_scope.get_simple_for_line(row) + print 'complete_scope', scope + print 'user_scope', f.parser.user_scope.get_simple_for_line(row) try: path = f.get_row_path(column) @@ -184,27 +184,24 @@ def complete(source, row, column, file_callback=None): if path and path[0]: # just parse one statement r = parsing.PyFuzzyParser(".".join(path)) - print 'p', r.top.get_code().replace('\n', r'\n'), r.top.statements[0] - evaluate.follow_statement(r.top.statements[0], scope) - exit() + #print 'p', r.top.get_code().replace('\n', r'\n'), r.top.statements[0] + scopes = evaluate.follow_statement(r.top.statements[0], scope) - name = path.pop() - if path: - scopes = evaluate.follow_path(scope, tuple(path)) + #name = path.pop() # use this later + compl = [] + debug.dbg('possible scopes') + for s in scopes: + compl += s.get_defined_names() - debug.dbg('possible scopes', scopes) - compl = [] - for s in scopes: - compl += s.get_defined_names() - - else: - compl = evaluate.get_names_for_scope(scope) + #else: + # compl = evaluate.get_names_for_scope(scope) debug.dbg('possible-compl', compl) # make a partial comparison, because the other options have to # be returned as well. - result = [c for c in compl if name in c.names[-1]] + result = compl + #result = [c for c in compl if name in c.names[-1]] return result diff --git a/modules.py b/modules.py index 11873fc2..2b01f3ff 100644 --- a/modules.py +++ b/modules.py @@ -3,11 +3,13 @@ import sys import debug import parsing +import builtin files = {} load_module_cb = None module_find_path = sys.path[1:] +# TODO we need module caching class ModuleNotFound(Exception): pass @@ -48,20 +50,6 @@ class File(object): else: return None -class BuiltinModule: - def __init__(self, name): - self.name = name - self.content = {} - exec 'import %s as module' % name in self.content - self.module = self.content['module'] - - @property - def docstr(self): - # TODO get the help string, not just the docstr - return self.module.__doc__ - - def get_defined_names(self): - return dir(self.module) def find_module(point_path): """ @@ -103,7 +91,6 @@ def find_module(point_path): if current_namespace[0]: f = File(current_namespace[2], current_namespace[0].read()) - scope = f.parser.top else: - scope = BuiltinModule(current_namespace[1]) - return scope, rest + f = builtin.Parser(current_namespace[1]) + return f.parser.top, rest diff --git a/parsing.py b/parsing.py index 4060368d..98c04260 100644 --- a/parsing.py +++ b/parsing.py @@ -146,6 +146,15 @@ class Scope(Simple): self.imports.append(imp) imp.parent = self + def get_imports(self): + """ Gets also the imports within flow statements """ + i = self.imports + for s in self.statements: + if isinstance(s, Scope): + i += s.get_imports() + print 'geti', i + return i + def add_global(self, name): """ Global means in these context a function (subscope) which has a global @@ -198,12 +207,13 @@ class Scope(Simple): n += self.global_vars for i in self.imports: - n += i.get_names() + if not i.star: + n += i.get_names() return n def get_defined_names(self): - return [n for n in self.get_set_vars() if len(n) == 1] + return [n for n in self.get_set_vars() if isinstance(n, Import) or len(n) == 1] def is_empty(self): """ @@ -1124,7 +1134,7 @@ class PyFuzzyParser(object): while indent <= self.scope.indent \ and token_type in [tokenize.NAME] \ and self.scope != self.top: - debug.warning('syntax_err, dedent @%s - %s<=%s', \ + debug.warning('syntax error: dedent @%s - %s<=%s', \ (self.line_nr, indent, self.scope.indent)) self.scope.line_end = self.line_nr self.scope = self.scope.parent diff --git a/test.py b/test.py index a700e6cf..20b46d32 100644 --- a/test.py +++ b/test.py @@ -44,7 +44,7 @@ class Supi(A, datetime.datetime): class A(): class B(): def test(self): - return a.test() + return 8 + a.test() class init: def __init__(self, a, b=3, c = A): self.a = a @@ -130,7 +130,7 @@ import time class c1(): c2,c5 = c1(), c1().c3() def c3(self): - from parsing import Scope as c4 + import tokenize as c4 #from parsing import Scope as c4 c5 = c4 c5 = 1 return c5+1+'asdf' @@ -142,7 +142,7 @@ c, 1, c3()) [0].pop() c = "a".join([1,2]) c = c1().c3().sleep() - asdf = c1; asdf2 = asdf b= asdf2 -c = b().c3().F_OK +#import parsing as test +c = b().c3()