import work - including star imports

builtin.py

@@ -0,0 +1,161 @@
+import re
+
+import debug
+import parsing
+
+
+class Parser(object):
+    map_types = {
+        'floating point number': '0.0',
+        'string': '""',
+        'str': '""',
+        'integer': '0',
+        'int': '0',
+        'dictionary': '{}',
+        'list': '[]',
+        'object': '{}',
+        # TODO things like dbg: ('not working', 'tuple of integers')
+    }
+
+    """ This module tries to imitate parsing.Scope """
+    def __init__(self, name):
+        self.name = name
+        self.parent = None
+        self.content = {}
+        exec 'import %s as module' % name in self.content
+        self.module = self.content['module']
+        self._parser = None
+
+    @property
+    def parser(self):
+        """ get the parser lazy """
+        if self._parser:
+            return self._parser
+        else:
+            code = self.generate_code(self.module)
+            try:
+                self._parser = parsing.PyFuzzyParser(code)
+            except:
+                debug.warning('not possible to resolve', self.name, code)
+                #open('builtin_fail', 'w').write(code)
+                raise
+            return self._parser
+
+    def generate_code(self, scope, depth=0):
+        """
+        Generate a string, which uses python syntax as an input to the
+        PyFuzzyParser.
+        """
+        def get_types(names):
+            classes = {}
+            funcs = {}
+            stmts = {}
+            members = {}
+            for n in names:
+                if '__' in n:
+                    continue
+                # this has a builtin_function_or_method
+                exe = getattr(scope, n)
+                if type(exe).__name__ == 'builtin_function_or_method':
+                    funcs[n] = exe
+                elif type(exe) == type:
+                    classes[n] = exe
+                elif type(exe).__name__ == 'member_descriptor':
+                    members[n] = exe
+                else:
+                    stmts[n] = exe
+            return classes, funcs, stmts, members
+
+        code = ''
+        try:
+            try:
+                path = scope.__file__
+            except:
+                path = '?'
+            code += '# Generated module %s from %s\n' % (scope.__name__, path)
+        except:
+            pass
+        code += '"""\n%s\n"""\n' % scope.__doc__
+
+        names = set(dir(scope)) - {'__file__', '__name__', '__doc__',
+                                    '__path__', '__package__'}
+        classes, funcs, stmts, members = get_types(names)
+        #print 'blub', get_types(names)
+
+        # classes
+        for name, cl in classes.iteritems():
+            bases = (c.__name__ for c in cl.__bases__)
+            code += 'class %s(%s):\n' % (name, ','.join(bases))
+            if depth == 0:
+                cl_code = self.generate_code(cl, depth + 1)
+                code += parsing.indent_block(cl_code)
+            code += '\n'
+
+        # functions
+        for name, func in funcs.iteritems():
+            params, ret = parse_function_doc(func)
+            code += 'def %s(%s):\n' % (name, params)
+            block = '"""\n%s\n"""\n' % func.__doc__
+            block += '%s\n\n' % ret
+            code += parsing.indent_block(block)
+
+        # class members (functions)
+        for name, func in members.iteritems():
+            ret = 'pass'
+            code += '@property\ndef %s(self):\n' % (name)
+            block = '"""\n%s\n"""\n' % func.__doc__
+            block += '%s\n\n' % ret
+            code += parsing.indent_block(block)
+
+        # variables
+        for name, value in stmts.iteritems():
+            if isinstance(value, str):
+                value = repr(value)
+            elif type(value) == file:
+                value = 'file'
+            code += '%s = %s\n' % (name, value)
+
+        import sys
+        #if depth == 0:
+        #    sys.stdout.write(code)
+        #    exit()
+        return code
+
+
+def parse_function_doc(func):
+    """
+    Takes a function and returns the params and return value as a tuple.
+    This is nothing more than a docstring parser.
+    """
+    # TODO: things like utime(path, (atime, mtime)) and a(b [, b]) -> None
+    params = []
+    doc = func.__doc__
+    end = doc.index(')')
+    #print 'blubedi', doc
+    param_str = doc[doc.index('(') + 1:end]
+    try:
+        index = doc.index('-> ', end, end + 7)
+    except ValueError:
+        ret = 'pass'
+    else:
+        pattern = re.compile(r'(,\n|[^\n])+')
+        ret_str = pattern.match(doc, index + 3).group(0)
+        ret = Parser.map_types.get(ret_str, ret_str)
+        if ret == ret_str and ret not in ['None', 'object', 'tuple', 'set']:
+            debug.dbg('not working', ret_str)
+        if ret != 'pass':
+            ret = 'return ' + ret
+    return param_str, ret
+
+"""if current.arr_type == parsing.Array.EMPTY:
+    # the normal case - no array type
+    print 'length', len(current)
+elif current.arr_type == parsing.Array.LIST:
+    result.append(__builtin__.list())
+elif current.arr_type == parsing.Array.SET:
+    result.append(__builtin__.set())
+elif current.arr_type == parsing.Array.TUPLE:
+    result.append(__builtin__.tuple())
+elif current.arr_type == parsing.Array.DICT:
+    result.append(__builtin__.dict())
+    """

debug.py

@@ -1,15 +1,27 @@
 import inspect
 
+NOTICE = object()
+WARNING = object()
+ERROR = object()
+
 def dbg(*args):
     if debug_function:
         frm =  inspect.stack()[1]
         mod = inspect.getmodule(frm[0])
         if not (mod.__name__ in ignored_modules):
-            debug_function(*args)
+            debug_function(NOTICE, *args)
 
 def warning(*args):
     if debug_function:
-        debug_function(*args)
+        debug_function(WARNING, *args)
 
-debug_function = None
+def error(*args):
+    if debug_function:
+        debug_function(ERROR, *args)
+
+def print_to_stdout(level, *args):
+    """ The default debug function """
+    print 'dbg:' if level == NOTICE else 'warning:', args
+
+debug_function = print_to_stdout
 ignored_modules = []

evaluate.py

@@ -1,3 +1,9 @@
+"""
+follow_statement -> follow_call -> follow_paths -> follow_path
+'follow_import'
+
+`get_names_for_scope` and `get_scopes_for_name` are search functions
+"""
 import itertools
 
 import parsing
@@ -8,6 +14,7 @@ import debug
 class Exec(object):
     def __init__(self, base):
         self.base = base
+
     def get_parent_until(self, *args):
         return self.base.get_parent_until(*args)
 
@@ -46,49 +53,39 @@ class Execution(Exec):
     """
     This class is used to evaluate functions and their returns.
     """
-
     cache = {}
+
     def get_return_types(self):
         """
         Get the return vars of a function.
         """
         def remove_executions(scope, get_returns=False):
-            if isinstance(scope, Execution):
+            stmts = []
+            if isinstance(scope, parsing.Class):
                 # there maybe executions of executions
-                stmts = scope.get_return_types()
+                stmts = [Instance(scope)]
             else:
                 if get_returns:
-                    stmts = scope.returns
+                    ret = scope.returns
+                    for s in ret:
+                        for stmt in  follow_statement(s):
+                            stmts += remove_executions(stmt)
                 else:
-                    stmts = [scope]
+                    stmts.append(scope)
             return stmts
 
         # check cache
         try:
+            debug.dbg('hit function cache', self.base)
             return Execution.cache[self.base]
         except KeyError:
             # cache is not only here as a cache, but also to prevent an
             # endless recursion.
             Execution.cache[self.base] = []
 
-        result = []
+        result = remove_executions(self.base, True)
-        stmts = remove_executions(self.base, True)
+        debug.dbg('exec stmts=', result, self.base, repr(self))
-        print 'stmts=', stmts, self.base, repr(self)
 
-        #n += self.function.get_set_vars()
-        # these are the statements of the return functions
-        for stmt in stmts:
-            if isinstance(stmt, parsing.Class):
-                # it might happen, that a function returns a Class and this
-                # gets executed, therefore get the instance here.
-                result.append(Instance(stmt))
-            else:
-                print 'addstmt', stmt
-                for followed in follow_statement(stmt):
-                    print 'followed', followed
-                    result += remove_executions(followed)
-
-            print 'ret', stmt
         Execution.cache[self.base] = result
         return result
 
@@ -106,11 +103,10 @@ def get_names_for_scope(scope):
         if not isinstance(scope, parsing.Class) or scope == start_scope:
             compl += scope.get_set_vars()
         scope = scope.parent
-    print 'get_names_for_scope', scope, len(compl)
     return compl
 
 
-def get_scopes_for_name(scope, name, search_global=False, search_func=None):
+def get_scopes_for_name(scope, name, search_global=False):
     """
     :return: List of Names. Their parents are the scopes, they are defined in.
     :rtype: list
@@ -129,6 +125,7 @@ def get_scopes_for_name(scope, name, search_global=False, search_func=None):
                 res_new += remove_statements(scopes)
             else:
                 res_new.append(r)
+        debug.dbg('sfn remove', res_new, result)
         return res_new
 
     def filter_name(scopes):
@@ -137,6 +134,7 @@ def get_scopes_for_name(scope, name, search_global=False, search_func=None):
         for scope in scopes:
             if isinstance(scope, parsing.Import):
                 try:
+                    debug.dbg('star import', scope)
                     i = follow_import(scope).get_defined_names()
                 except modules.ModuleNotFound:
                     debug.dbg('StarImport not found: ' + str(scope))
@@ -145,20 +143,40 @@ def get_scopes_for_name(scope, name, search_global=False, search_func=None):
             else:
                 if [name] == list(scope.names):
                     result.append(scope.parent)
+        debug.dbg('sfn filter', result)
         return result
 
-    if search_func:
+    if search_global:
-        names = search_func()
-    elif search_global:
         names = get_names_for_scope(scope)
     else:
         names = scope.get_set_vars()
 
-    # TODO here are the star imports handled, we need to get the names here.
-    # This means things like from pylab import *
     return remove_statements(filter_name(names))
 
 
+def resolve_results(scopes):
+    """ Here we follow the results - to get what we really want """
+    result = []
+    for s in scopes:
+        if isinstance(s, parsing.Import):
+            print 'dini mueter, steile griech!'
+            try:
+                scope = follow_import(s)
+                #for r in resolve_results([follow_import(s)]):
+                #    if isinstance(r, parsing.Import):
+                #        resolve_results(r)
+                #    else:
+                #        resolve
+            except modules.ModuleNotFound:
+                debug.dbg('Module not found: ' + str(s))
+            else:
+                result.append(scope)
+                result += resolve_results(i for i in scope.get_imports() if i.star)
+        else:
+            result.append(s)
+    return result
+
+
 def follow_statement(stmt, scope=None):
     """
     :param stmt: contains a statement
@@ -168,41 +186,28 @@ def follow_statement(stmt, scope=None):
         scope = stmt.get_parent_until(parsing.Function)
     result = []
     calls = stmt.get_assignment_calls()
-    print 'calls', calls, calls.values
+    debug.dbg('calls', calls, calls.values)
     for tokens in calls:
         for tok in tokens:
-            print 'tok', tok, type(tok), isinstance(tok,str)
             if not isinstance(tok, str):
                 # the string tokens are just operations (+, -, etc.)
                 result += follow_call(scope, tok)
     return result
 
+
 def follow_call(scope, call):
+    """ Follow a call is following a function, variable, string, etc. """
     path = call.generate_call_list()
 
     current = next(path)
-    result = []
     if isinstance(current, parsing.Array):
-        """if current.arr_type == parsing.Array.EMPTY:
+        result = [current]
-            # the normal case - no array type
-            print 'length', len(current)
-        elif current.arr_type == parsing.Array.LIST:
-            result.append(__builtin__.list())
-        elif current.arr_type == parsing.Array.SET:
-            result.append(__builtin__.set())
-        elif current.arr_type == parsing.Array.TUPLE:
-            result.append(__builtin__.tuple())
-        elif current.arr_type == parsing.Array.DICT:
-            result.append(__builtin__.dict())
-            """
-        result.append(current)
     else:
-        result = get_scopes_for_name(scope, current, search_global=True)
+        scopes = get_scopes_for_name(scope, current, search_global=True)
-        pass
+        result = resolve_results(scopes)
 
-    print 'before', result
+    debug.dbg('call before', result, current, scope)
     result = follow_paths(path, result)
-    print 'after result', result
 
     return result
 
@@ -210,12 +215,11 @@ def follow_call(scope, call):
 def follow_paths(path, results):
     results_new = []
     try:
-        if len(results) > 1:
+        if results:
-            iter_paths = itertools.tee(path, len(results))
+            if len(results) > 1:
-        else:
+                iter_paths = itertools.tee(path, len(results))
-            iter_paths = [path]
+            else:
-        print 'enter', results, len(results)
+                iter_paths = [path]
-        if len(results):
             for i, r in enumerate(results):
                 results_new += follow_path(iter_paths[i], r)
     except StopIteration:
@@ -225,21 +229,11 @@ def follow_paths(path, results):
 
 def follow_path(path, input):
     """
-    takes a generator and tries to complete the path
+    Takes a generator and tries to complete the path.
     """
-    def add_results(scopes):
+    # current is either an Array or a Scope
-        """ Here we follow the results - to get what we really want """
+    current = next(path)
-        result = []
+    debug.dbg('follow', current, input)
-        for s in scopes:
-            if isinstance(s, parsing.Import):
-                print 'dini mueter, steile griech!'
-                try:
-                    result.append(follow_import(s))
-                except modules.ModuleNotFound:
-                    debug.dbg('Module not found: ' + str(s))
-            else:
-                result.append(s)
-        return result
 
     def filter_result(scope):
         result = []
@@ -247,30 +241,24 @@ def follow_path(path, input):
             # this must be an execution, either () or []
             if current.arr_type == parsing.Array.LIST:
                 result = []  # TODO eval lists
-            else:
+            elif current.arr_type not in [parsing.Array.DICT, parsing]:
                 # scope must be a class or func - make an instance or execution
-                if isinstance(scope, parsing.Class):
+                debug.dbg('befexec', scope)
-                    result.append(Instance(scope))
+                result = resolve_results(Execution(scope).get_return_types())
-                else:
+                debug.dbg('exec', result)
-                    #try:
+                #except AttributeError:
-                    print '\n\n\n\n\nbefexec', scope
+                #    debug.dbg('cannot execute:', scope)
-                    stmts = add_results(Execution(scope).get_return_types())
+            else:
-                    debug.dbg('exec', stmts)
+                # curly braces are not allowed, because they make no sense
-                    result = stmts
+                debug.warning('strange function call with {}', current, scope)
-                    #except AttributeError:
-                    #    debug.dbg('cannot execute:', scope)
         else:
             if isinstance(scope, parsing.Function):
                 # TODO check default function methods and return them
                 result = []
             else:
                 # TODO check magic class methods and return them also
-                result = add_results(get_scopes_for_name(scope, current))
+                result = resolve_results(get_scopes_for_name(scope, current))
         return result
-
-    current = next(path)
-    print 'follow', input, current
-
     return follow_paths(path, filter_result(input))
 
 
@@ -293,4 +281,3 @@ def follow_import(_import):
 
     debug.dbg('after import', scope, rest)
     return scope
-

ftest.py

@@ -0,0 +1,13 @@
+#!/usr/bin/env python
+
+import functions
+
+#functions.debug.debug_function
+functions.debug.ignored_modules += ['parsing', 'functions']
+functions.modules.module_find_path.insert(0, '.')
+
+with open('test.py') as f:
+    code = f.read()
+completions = functions.complete(code, 50, 20)
+
+print '\n', [c.names for c in completions]

functions.py

@@ -171,8 +171,8 @@ def complete(source, row, column, file_callback=None):
     debug.dbg('-' * 70)
     debug.dbg(' ' * 62 + 'complete')
     debug.dbg('-' * 70)
-    print scope
+    print 'complete_scope', scope
-    print f.parser.user_scope.get_simple_for_line(row)
+    print 'user_scope', f.parser.user_scope.get_simple_for_line(row)
 
     try:
         path = f.get_row_path(column)
@@ -184,27 +184,24 @@ def complete(source, row, column, file_callback=None):
     if path and path[0]:
         # just parse one statement
         r = parsing.PyFuzzyParser(".".join(path))
-        print 'p', r.top.get_code().replace('\n', r'\n'), r.top.statements[0]
+        #print 'p', r.top.get_code().replace('\n', r'\n'), r.top.statements[0]
-        evaluate.follow_statement(r.top.statements[0], scope)
+        scopes = evaluate.follow_statement(r.top.statements[0], scope)
-        exit()
 
-        name = path.pop()
+        #name = path.pop() # use this later
-        if path:
+        compl = []
-            scopes = evaluate.follow_path(scope, tuple(path))
+        debug.dbg('possible scopes')
+        for s in scopes:
+            compl += s.get_defined_names()
 
-            debug.dbg('possible scopes', scopes)
+        #else:
-            compl = []
+        #    compl = evaluate.get_names_for_scope(scope)
-            for s in scopes:
-                compl += s.get_defined_names()
-
-        else:
-            compl = evaluate.get_names_for_scope(scope)
 
         debug.dbg('possible-compl', compl)
 
         # make a partial comparison, because the other options have to
         # be returned as well.
-        result = [c for c in compl if name in c.names[-1]]
+        result = compl
+        #result = [c for c in compl if name in c.names[-1]]
 
     return result
 

modules.py

@@ -3,11 +3,13 @@ import sys
 
 import debug
 import parsing
+import builtin
 
 files = {}
 load_module_cb = None
 module_find_path = sys.path[1:]
 
+# TODO we need module caching
 
 class ModuleNotFound(Exception):
     pass
@@ -48,20 +50,6 @@ class File(object):
         else:
             return None
 
-class BuiltinModule:
-    def __init__(self, name):
-        self.name = name
-        self.content = {}
-        exec 'import %s as module' % name in self.content
-        self.module = self.content['module']
-
-    @property
-    def docstr(self):
-        # TODO get the help string, not just the docstr
-        return self.module.__doc__
-
-    def get_defined_names(self):
-        return dir(self.module)
 
 def find_module(point_path):
     """
@@ -103,7 +91,6 @@ def find_module(point_path):
 
     if current_namespace[0]:
         f = File(current_namespace[2], current_namespace[0].read())
-        scope = f.parser.top
     else:
-        scope = BuiltinModule(current_namespace[1])
+        f = builtin.Parser(current_namespace[1])
-    return scope, rest
+    return f.parser.top, rest

parsing.py

@@ -146,6 +146,15 @@ class Scope(Simple):
         self.imports.append(imp)
         imp.parent = self
 
+    def get_imports(self):
+        """ Gets also the imports within flow statements """
+        i = self.imports
+        for s in self.statements:
+            if isinstance(s, Scope):
+                i += s.get_imports() 
+        print 'geti', i
+        return i 
+
     def add_global(self, name):
         """
         Global means in these context a function (subscope) which has a global
@@ -198,12 +207,13 @@ class Scope(Simple):
         n += self.global_vars
 
         for i in self.imports:
-            n += i.get_names()
+            if not i.star:
+                n += i.get_names()
 
         return n
 
     def get_defined_names(self):
-        return [n for n in self.get_set_vars() if len(n) == 1]
+        return [n for n in self.get_set_vars() if isinstance(n, Import) or len(n) == 1]
 
     def is_empty(self):
         """
@@ -1124,7 +1134,7 @@ class PyFuzzyParser(object):
                 while indent <= self.scope.indent \
                         and token_type in [tokenize.NAME] \
                         and self.scope != self.top:
-                    debug.warning('syntax_err, dedent @%s - %s<=%s', \
+                    debug.warning('syntax error: dedent @%s - %s<=%s', \
                             (self.line_nr, indent, self.scope.indent))
                     self.scope.line_end = self.line_nr
                     self.scope = self.scope.parent

test.py

@@ -44,7 +44,7 @@ class Supi(A, datetime.datetime):
 class A():
   class B():
   def test(self):
-    return a.test()
+    return 8 + a.test()
 class init:
     def __init__(self, a, b=3, c = A):
      self.a  =  a
@@ -130,7 +130,7 @@ import time
 class c1():
     c2,c5 = c1(), c1().c3()
     def c3(self): 
-        from parsing import Scope as c4
+        import tokenize as c4 #from parsing import Scope as c4
         c5 = c4
         c5 = 1
         return c5+1+'asdf'
@@ -142,7 +142,7 @@ c, 1, c3()) [0].pop()
 c = "a".join([1,2])
 
 c = c1().c3().sleep()
-
 asdf = c1; asdf2 = asdf
 b= asdf2
-c = b().c3().F_OK
+#import parsing as test
+c = b().c3()