import work - including star imports

debug.py

@@ -1,15 +1,27 @@
 import inspect
 
+NOTICE = object()
+WARNING = object()
+ERROR = object()
+
 def dbg(*args):
     if debug_function:
         frm =  inspect.stack()[1]
         mod = inspect.getmodule(frm[0])
         if not (mod.__name__ in ignored_modules):
-            debug_function(*args)
+            debug_function(NOTICE, *args)
 
 def warning(*args):
     if debug_function:
-        debug_function(*args)
+        debug_function(WARNING, *args)
 
-debug_function = None
+def error(*args):
+    if debug_function:
+        debug_function(ERROR, *args)
+
+def print_to_stdout(level, *args):
+    """ The default debug function """
+    print 'dbg:' if level == NOTICE else 'warning:', args
+
+debug_function = print_to_stdout
 ignored_modules = []

evaluate.py

@@ -1,3 +1,9 @@
+"""
+follow_statement -> follow_call -> follow_paths -> follow_path
+'follow_import'
+
+`get_names_for_scope` and `get_scopes_for_name` are search functions
+"""
 import itertools
 
 import parsing
@@ -8,6 +14,7 @@ import debug
 class Exec(object):
     def __init__(self, base):
         self.base = base
+
     def get_parent_until(self, *args):
         return self.base.get_parent_until(*args)
 
@@ -46,49 +53,39 @@ class Execution(Exec):
     """
     This class is used to evaluate functions and their returns.
     """
-
     cache = {}
+
     def get_return_types(self):
         """
         Get the return vars of a function.
         """
         def remove_executions(scope, get_returns=False):
-            if isinstance(scope, Execution):
+            stmts = []
+            if isinstance(scope, parsing.Class):
                 # there maybe executions of executions
-                stmts = scope.get_return_types()
+                stmts = [Instance(scope)]
             else:
                 if get_returns:
-                    stmts = scope.returns
+                    ret = scope.returns
+                    for s in ret:
+                        for stmt in  follow_statement(s):
+                            stmts += remove_executions(stmt)
                 else:
-                    stmts = [scope]
+                    stmts.append(scope)
             return stmts
 
         # check cache
         try:
+            debug.dbg('hit function cache', self.base)
             return Execution.cache[self.base]
         except KeyError:
             # cache is not only here as a cache, but also to prevent an
             # endless recursion.
             Execution.cache[self.base] = []
 
-        result = []
-        stmts = remove_executions(self.base, True)
-        print 'stmts=', stmts, self.base, repr(self)
+        result = remove_executions(self.base, True)
+        debug.dbg('exec stmts=', result, self.base, repr(self))
 
-        #n += self.function.get_set_vars()
-        # these are the statements of the return functions
-        for stmt in stmts:
-            if isinstance(stmt, parsing.Class):
-                # it might happen, that a function returns a Class and this
-                # gets executed, therefore get the instance here.
-                result.append(Instance(stmt))
-            else:
-                print 'addstmt', stmt
-                for followed in follow_statement(stmt):
-                    print 'followed', followed
-                    result += remove_executions(followed)
-
-            print 'ret', stmt
         Execution.cache[self.base] = result
         return result
 
@@ -106,11 +103,10 @@ def get_names_for_scope(scope):
         if not isinstance(scope, parsing.Class) or scope == start_scope:
             compl += scope.get_set_vars()
         scope = scope.parent
-    print 'get_names_for_scope', scope, len(compl)
     return compl
 
 
-def get_scopes_for_name(scope, name, search_global=False, search_func=None):
+def get_scopes_for_name(scope, name, search_global=False):
     """
     :return: List of Names. Their parents are the scopes, they are defined in.
     :rtype: list
@@ -129,6 +125,7 @@ def get_scopes_for_name(scope, name, search_global=False, search_func=None):
                 res_new += remove_statements(scopes)
             else:
                 res_new.append(r)
+        debug.dbg('sfn remove', res_new, result)
         return res_new
 
     def filter_name(scopes):
@@ -137,6 +134,7 @@ def get_scopes_for_name(scope, name, search_global=False, search_func=None):
         for scope in scopes:
             if isinstance(scope, parsing.Import):
                 try:
+                    debug.dbg('star import', scope)
                     i = follow_import(scope).get_defined_names()
                 except modules.ModuleNotFound:
                     debug.dbg('StarImport not found: ' + str(scope))
@@ -145,20 +143,40 @@ def get_scopes_for_name(scope, name, search_global=False, search_func=None):
             else:
                 if [name] == list(scope.names):
                     result.append(scope.parent)
+        debug.dbg('sfn filter', result)
         return result
 
-    if search_func:
-        names = search_func()
-    elif search_global:
+    if search_global:
         names = get_names_for_scope(scope)
     else:
         names = scope.get_set_vars()
 
-    # TODO here are the star imports handled, we need to get the names here.
-    # This means things like from pylab import *
     return remove_statements(filter_name(names))
 
 
+def resolve_results(scopes):
+    """ Here we follow the results - to get what we really want """
+    result = []
+    for s in scopes:
+        if isinstance(s, parsing.Import):
+            print 'dini mueter, steile griech!'
+            try:
+                scope = follow_import(s)
+                #for r in resolve_results([follow_import(s)]):
+                #    if isinstance(r, parsing.Import):
+                #        resolve_results(r)
+                #    else:
+                #        resolve
+            except modules.ModuleNotFound:
+                debug.dbg('Module not found: ' + str(s))
+            else:
+                result.append(scope)
+                result += resolve_results(i for i in scope.get_imports() if i.star)
+        else:
+            result.append(s)
+    return result
+
+
 def follow_statement(stmt, scope=None):
     """
     :param stmt: contains a statement
@@ -168,41 +186,28 @@ def follow_statement(stmt, scope=None):
         scope = stmt.get_parent_until(parsing.Function)
     result = []
     calls = stmt.get_assignment_calls()
-    print 'calls', calls, calls.values
+    debug.dbg('calls', calls, calls.values)
     for tokens in calls:
         for tok in tokens:
-            print 'tok', tok, type(tok), isinstance(tok,str)
             if not isinstance(tok, str):
                 # the string tokens are just operations (+, -, etc.)
                 result += follow_call(scope, tok)
     return result
 
+
 def follow_call(scope, call):
+    """ Follow a call is following a function, variable, string, etc. """
     path = call.generate_call_list()
 
     current = next(path)
-    result = []
     if isinstance(current, parsing.Array):
-        """if current.arr_type == parsing.Array.EMPTY:
-            # the normal case - no array type
-            print 'length', len(current)
-        elif current.arr_type == parsing.Array.LIST:
-            result.append(__builtin__.list())
-        elif current.arr_type == parsing.Array.SET:
-            result.append(__builtin__.set())
-        elif current.arr_type == parsing.Array.TUPLE:
-            result.append(__builtin__.tuple())
-        elif current.arr_type == parsing.Array.DICT:
-            result.append(__builtin__.dict())
-            """
-        result.append(current)
+        result = [current]
     else:
-        result = get_scopes_for_name(scope, current, search_global=True)
-        pass
+        scopes = get_scopes_for_name(scope, current, search_global=True)
+        result = resolve_results(scopes)
 
-    print 'before', result
+    debug.dbg('call before', result, current, scope)
     result = follow_paths(path, result)
-    print 'after result', result
 
     return result
 
@@ -210,12 +215,11 @@ def follow_call(scope, call):
 def follow_paths(path, results):
     results_new = []
     try:
-        if len(results) > 1:
-            iter_paths = itertools.tee(path, len(results))
-        else:
-            iter_paths = [path]
-        print 'enter', results, len(results)
-        if len(results):
+        if results:
+            if len(results) > 1:
+                iter_paths = itertools.tee(path, len(results))
+            else:
+                iter_paths = [path]
             for i, r in enumerate(results):
                 results_new += follow_path(iter_paths[i], r)
     except StopIteration:
@@ -225,21 +229,11 @@ def follow_paths(path, results):
 
 def follow_path(path, input):
     """
-    takes a generator and tries to complete the path
+    Takes a generator and tries to complete the path.
     """
-    def add_results(scopes):
-        """ Here we follow the results - to get what we really want """
-        result = []
-        for s in scopes:
-            if isinstance(s, parsing.Import):
-                print 'dini mueter, steile griech!'
-                try:
-                    result.append(follow_import(s))
-                except modules.ModuleNotFound:
-                    debug.dbg('Module not found: ' + str(s))
-            else:
-                result.append(s)
-        return result
+    # current is either an Array or a Scope
+    current = next(path)
+    debug.dbg('follow', current, input)
 
     def filter_result(scope):
         result = []
@@ -247,30 +241,24 @@ def follow_path(path, input):
             # this must be an execution, either () or []
             if current.arr_type == parsing.Array.LIST:
                 result = []  # TODO eval lists
-            else:
+            elif current.arr_type not in [parsing.Array.DICT, parsing]:
                 # scope must be a class or func - make an instance or execution
-                if isinstance(scope, parsing.Class):
-                    result.append(Instance(scope))
-                else:
-                    #try:
-                    print '\n\n\n\n\nbefexec', scope
-                    stmts = add_results(Execution(scope).get_return_types())
-                    debug.dbg('exec', stmts)
-                    result = stmts
-                    #except AttributeError:
-                    #    debug.dbg('cannot execute:', scope)
+                debug.dbg('befexec', scope)
+                result = resolve_results(Execution(scope).get_return_types())
+                debug.dbg('exec', result)
+                #except AttributeError:
+                #    debug.dbg('cannot execute:', scope)
+            else:
+                # curly braces are not allowed, because they make no sense
+                debug.warning('strange function call with {}', current, scope)
         else:
             if isinstance(scope, parsing.Function):
                 # TODO check default function methods and return them
                 result = []
             else:
                 # TODO check magic class methods and return them also
-                result = add_results(get_scopes_for_name(scope, current))
+                result = resolve_results(get_scopes_for_name(scope, current))
         return result
-
-    current = next(path)
-    print 'follow', input, current
-
     return follow_paths(path, filter_result(input))
 
 
@@ -293,4 +281,3 @@ def follow_import(_import):
 
     debug.dbg('after import', scope, rest)
     return scope
-

ftest.py

@@ -0,0 +1,13 @@
+#!/usr/bin/env python
+
+import functions
+
+#functions.debug.debug_function
+functions.debug.ignored_modules += ['parsing', 'functions']
+functions.modules.module_find_path.insert(0, '.')
+
+with open('test.py') as f:
+    code = f.read()
+completions = functions.complete(code, 50, 20)
+
+print '\n', [c.names for c in completions]

functions.py

@@ -171,8 +171,8 @@ def complete(source, row, column, file_callback=None):
     debug.dbg('-' * 70)
     debug.dbg(' ' * 62 + 'complete')
     debug.dbg('-' * 70)
-    print scope
-    print f.parser.user_scope.get_simple_for_line(row)
+    print 'complete_scope', scope
+    print 'user_scope', f.parser.user_scope.get_simple_for_line(row)
 
     try:
         path = f.get_row_path(column)
@@ -184,27 +184,24 @@ def complete(source, row, column, file_callback=None):
     if path and path[0]:
         # just parse one statement
         r = parsing.PyFuzzyParser(".".join(path))
-        print 'p', r.top.get_code().replace('\n', r'\n'), r.top.statements[0]
-        evaluate.follow_statement(r.top.statements[0], scope)
-        exit()
+        #print 'p', r.top.get_code().replace('\n', r'\n'), r.top.statements[0]
+        scopes = evaluate.follow_statement(r.top.statements[0], scope)
 
-        name = path.pop()
-        if path:
-            scopes = evaluate.follow_path(scope, tuple(path))
+        #name = path.pop() # use this later
+        compl = []
+        debug.dbg('possible scopes')
+        for s in scopes:
+            compl += s.get_defined_names()
 
-            debug.dbg('possible scopes', scopes)
-            compl = []
-            for s in scopes:
-                compl += s.get_defined_names()
-
-        else:
-            compl = evaluate.get_names_for_scope(scope)
+        #else:
+        #    compl = evaluate.get_names_for_scope(scope)
 
         debug.dbg('possible-compl', compl)
 
         # make a partial comparison, because the other options have to
         # be returned as well.
-        result = [c for c in compl if name in c.names[-1]]
+        result = compl
+        #result = [c for c in compl if name in c.names[-1]]
 
     return result
 

modules.py

@@ -3,11 +3,13 @@ import sys
 
 import debug
 import parsing
+import builtin
 
 files = {}
 load_module_cb = None
 module_find_path = sys.path[1:]
 
+# TODO we need module caching
 
 class ModuleNotFound(Exception):
     pass
@@ -48,20 +50,6 @@ class File(object):
         else:
             return None
 
-class BuiltinModule:
-    def __init__(self, name):
-        self.name = name
-        self.content = {}
-        exec 'import %s as module' % name in self.content
-        self.module = self.content['module']
-
-    @property
-    def docstr(self):
-        # TODO get the help string, not just the docstr
-        return self.module.__doc__
-
-    def get_defined_names(self):
-        return dir(self.module)
 
 def find_module(point_path):
     """
@@ -103,7 +91,6 @@ def find_module(point_path):
 
     if current_namespace[0]:
         f = File(current_namespace[2], current_namespace[0].read())
-        scope = f.parser.top
     else:
-        scope = BuiltinModule(current_namespace[1])
-    return scope, rest
+        f = builtin.Parser(current_namespace[1])
+    return f.parser.top, rest

parsing.py

@@ -146,6 +146,15 @@ class Scope(Simple):
         self.imports.append(imp)
         imp.parent = self
 
+    def get_imports(self):
+        """ Gets also the imports within flow statements """
+        i = self.imports
+        for s in self.statements:
+            if isinstance(s, Scope):
+                i += s.get_imports() 
+        print 'geti', i
+        return i 
+
     def add_global(self, name):
         """
         Global means in these context a function (subscope) which has a global
@@ -198,12 +207,13 @@ class Scope(Simple):
         n += self.global_vars
 
         for i in self.imports:
-            n += i.get_names()
+            if not i.star:
+                n += i.get_names()
 
         return n
 
     def get_defined_names(self):
-        return [n for n in self.get_set_vars() if len(n) == 1]
+        return [n for n in self.get_set_vars() if isinstance(n, Import) or len(n) == 1]
 
     def is_empty(self):
         """
@@ -1124,7 +1134,7 @@ class PyFuzzyParser(object):
                 while indent <= self.scope.indent \
                         and token_type in [tokenize.NAME] \
                         and self.scope != self.top:
-                    debug.warning('syntax_err, dedent @%s - %s<=%s', \
+                    debug.warning('syntax error: dedent @%s - %s<=%s', \
                             (self.line_nr, indent, self.scope.indent))
                     self.scope.line_end = self.line_nr
                     self.scope = self.scope.parent

test.py

@@ -44,7 +44,7 @@ class Supi(A, datetime.datetime):
 class A():
   class B():
   def test(self):
-    return a.test()
+    return 8 + a.test()
 class init:
     def __init__(self, a, b=3, c = A):
      self.a  =  a
@@ -130,7 +130,7 @@ import time
 class c1():
     c2,c5 = c1(), c1().c3()
     def c3(self): 
-        from parsing import Scope as c4
+        import tokenize as c4 #from parsing import Scope as c4
         c5 = c4
         c5 = 1
         return c5+1+'asdf'
@@ -142,7 +142,7 @@ c, 1, c3()) [0].pop()
 c = "a".join([1,2])
 
 c = c1().c3().sleep()
-
 asdf = c1; asdf2 = asdf
 b= asdf2
-c = b().c3().F_OK
+#import parsing as test
+c = b().c3()