added recursion decorator / fast_parent_copy fix

dynamic.py

@@ -128,15 +128,21 @@ counter = 0
 def dec(func):
     def wrapper(*args, **kwargs):
         global counter
-        a = args[0]._array.parent_stmt()
-        print '  '*counter + 'recursion,', a, id(a)
+        element = args[0]
+        if isinstance(element, evaluate.Array):
+            stmt = element._array.parent_stmt()
+        else:
+            # must be instance
+            stmt = element.var_args.parent_stmt()
+        print '  '*counter + 'recursion,', stmt
         counter += 1
         res = func(*args, **kwargs)
         counter -= 1
-        print '  '*counter + 'end,', args[0]
+        #print '  '*counter + 'end,'
         return res
     return wrapper
 
+#@dec
 @evaluate.memoize_default([])
 def _check_array_additions(compare_array, module, is_list):
     """
@@ -173,15 +179,7 @@ def _check_array_additions(compare_array, module, is_list):
             position = c.parent_stmt().start_pos
             scope = c.parent_stmt().parent()
 
-            # Special assignments should not be evaluated in this case. This
-            # would cause big recursion problems, because in cases like the
-            # code of jedi itself, += something is called and this call leads
-            # to many other things including params, which are not defined.
-            # This would lead again to dynamic param completion, and so on.
-            # In the end the definition is needed, and that's not with `+=`.
-            settings.evaluate_special_assignments = False
             found = evaluate.follow_call_path(backtrack_path, scope, position)
-            settings.evaluate_special_assignments = True
             if not compare_array in found:
                 continue
 

evaluate.py

@@ -30,7 +30,6 @@ import builtin
 import imports
 import helpers
 import dynamic
-import settings
 
 memoize_caches = []
 statement_path = []
@@ -77,6 +76,7 @@ def clear_caches():
         m.clear()
 
     dynamic.search_param_cache.clear()
+    helpers.ExecutionRecursionDecorator.reset()
 
     # memorize_caches must never be deleted, because the dicts will get lost in
     # the wrappers.
@@ -440,6 +440,7 @@ class Execution(Executable):
     multiple call to functions and recursion has to be avoided.
     """
     @memoize_default(default=[])
+    @helpers.ExecutionRecursionDecorator
     def get_return_types(self, evaluate_generator=False):
         """
         Get the return vars of a function.
@@ -645,10 +646,7 @@ class Execution(Executable):
         attr = getattr(self.base, prop)
         objects = []
         for element in attr:
-            temp, element.parent = element.parent, None
-            #copied = copy.deepcopy(element)
             copied = helpers.fast_parent_copy(element)
-            element.parent = temp
             copied.parent = weakref.ref(self)
             if isinstance(copied, parsing.Function):
                 copied = Function(copied)
@@ -709,7 +707,6 @@ class Generator(parsing.Base):
 
     def iter_content(self):
         """ returns the content of __iter__ """
-        #print self, follow_statement.node_statements()
         return Execution(self.func, self.var_args).get_return_types(True)
 
     def get_index_types(self, index=None):
@@ -1145,11 +1142,6 @@ def follow_statement(stmt, seek_name=None):
     """
     :param stmt: contains a statement
     """
-    if not settings.evaluate_special_assignments:
-        det = stmt.assignment_details
-        if det and det[0][0] != '=':
-            return []
-
     statement_path.append(stmt)  # important to know for the goto function
 
     debug.dbg('follow_stmt %s (%s)' % (stmt, seek_name))

helpers.py

@@ -5,6 +5,7 @@ import parsing
 import evaluate
 import debug
 import builtin
+import settings
 
 
 class RecursionDecorator(object):
@@ -14,6 +15,7 @@ class RecursionDecorator(object):
         self.reset()
 
     def __call__(self, stmt, *args, **kwargs):
+        #print stmt, len(self.node_statements())
         if self.push_stmt(stmt):
             return []
         else:
@@ -75,6 +77,65 @@ class RecursionNode(object):
                     and not self.is_ignored and not other.is_ignored
 
 
+class ExecutionRecursionDecorator(object):
+    """
+    Catches recursions of executions.
+    It is designed like a Singelton. Only one instance should exist.
+    """
+    def __init__(self, func):
+        self.func = func
+        self.reset()
+
+    def __call__(self, execution, evaluate_generator=False):
+        #print execution, self.recursion_level, self.execution_count,
+        #print len(self.execution_funcs),
+        a= self.check_recursion(execution, evaluate_generator)
+        #print a
+        if a:
+            result = []
+        else:
+            result = self.func(execution, evaluate_generator)
+        self.cleanup()
+        return result
+
+    @classmethod
+    def cleanup(cls):
+        cls.parent_execution_funcs.pop()
+        cls.recursion_level -= 1
+
+    @classmethod
+    def check_recursion(cls, execution, evaluate_generator):
+        in_par_execution_funcs = execution.base in cls.parent_execution_funcs
+        in_execution_funcs = execution.base in cls.execution_funcs
+        cls.recursion_level += 1
+        cls.execution_count += 1
+        cls.execution_funcs.add(execution.base)
+        cls.parent_execution_funcs.append(execution.base)
+
+        if isinstance(execution.base, (evaluate.Generator, evaluate.Array)):
+            return False
+        module = execution.get_parent_until()
+        if evaluate_generator or module == builtin.Builtin.scope:
+            return False
+
+        if in_par_execution_funcs:
+            if cls.recursion_level > settings.max_function_recursion_level:
+                return True
+        if in_execution_funcs and \
+                len(cls.execution_funcs) > settings.max_until_execution_unique:
+            return True
+        if cls.execution_count > settings.max_executions:
+            return True
+        return False
+
+    @classmethod
+    def reset(cls):
+        cls.recursion_level = 0
+        cls.parent_execution_funcs = []
+        cls.execution_funcs = set()
+        cls.execution_count = 0
+
+
 def fast_parent_copy(obj):
     """
     Much, much faster than deepcopy, but just for the elements in `classes`.
@@ -84,12 +145,6 @@ def fast_parent_copy(obj):
     def recursion(obj):
         new_obj = copy.copy(obj)
         new_elements[obj] = new_obj
-        if obj.parent is not None:
-            try:
-                new_obj.parent = weakref.ref(new_elements[obj.parent()])
-            except KeyError:
-                pass
-
         #print new_obj.__dict__
         for key, value in new_obj.__dict__.items():
             #if key in ['_parent_stmt', 'parent_stmt', '_parent', 'parent']: print key, value
@@ -99,6 +154,13 @@ def fast_parent_copy(obj):
                 new_obj.__dict__[key] = list_rec(value)
             elif isinstance(value, parsing.Simple):
                 new_obj.__dict__[key] = recursion(value)
+
+        if obj.parent is not None:
+            try:
+                new_obj.parent = weakref.ref(new_elements[obj.parent()])
+            except KeyError:
+                pass
+
         return new_obj
 
     def list_rec(list_obj):

parsing.py

@@ -391,9 +391,9 @@ class Flow(Scope):
     """
     def __init__(self, command, inits, start_pos, set_vars=None):
         self.next = None
+        self.command = command
         super(Flow, self).__init__(start_pos, '')
         self._parent = None
-        self.command = command
         # These have to be statements, because of with, which takes multiple.
         self.inits = inits
         for s in inits:
@@ -415,14 +415,6 @@ class Flow(Scope):
         if self.next:
             self.next.parent = value
 
-    def set_parent(self, value):
-        """
-        Normally this would be a setter, but since parents are normally
-        weakrefs (and therefore require execution),
-        I use a java like setter here.
-        """
-        self._parent = weakref.ref(value)
-
     def get_code(self, first_indent=False, indention="    "):
         if self.set_vars:
             vars = ",".join(map(lambda x: x.get_code(), self.set_vars))

settings.py

@@ -1,5 +1,5 @@
 # ----------------
-# global settings
+# dynamic stuff
 # ----------------
 
 dynamic_arrays_instances = True
@@ -7,8 +7,9 @@ dynamic_array_additions = True
 dynamic_params = True
 
 # ----------------
-# internally used:
+# recursions
 # ----------------
 
-# evaluation of +=, -=, /=, etc.
-evaluate_special_assignments = True
+max_function_recursion_level = 5
+max_until_execution_unique = 50
+max_executions = 5000

test/completion/basic.py

@@ -1,3 +1,27 @@
+# -----------------
+# if/else/elif
+# -----------------
+
+if 1:
+    1
+elif(3):
+    a = 3
+else:
+    a = ''
+#? int() str()
+a
+def func():
+    if 1:
+        1
+    elif(3):
+        a = 3
+    else:
+        a = ''
+    #? int() str()
+    return a
+#? int() str()
+func()
+
 # -----------------
 # for loops
 # -----------------

test/completion/thirdparty/jedi_.py

@@ -31,12 +31,19 @@ el = list(evaluate.get_names_for_scope(1))[0][1]
 #? list()
 el = list(evaluate.get_names_for_scope())[0][1]
 
-# TODO here should stand evaluate.Instance() and so on.
-# need to understand list comprehensions
-##? 
-el = list(evaluate.get_names_for_scope())[0][1][0]
-
 #? list()
 parsing.Scope((0,0)).get_set_vars()
-#? 
+#? parsing.Import() parsing.Name()
 parsing.Scope((0,0)).get_set_vars()[0]
+# TODO access parent is not possible, because that is not set in the class
+## parsing.Class()
+parsing.Scope((0,0)).get_set_vars()[0].parent
+
+#? parsing.Import() parsing.Name()
+el = list(evaluate.get_names_for_scope())[0][1][0]
+
+#? evaluate.Array() evaluate.Class() evaluate.Function() evaluate.Instance()
+list(evaluate.follow_call())[0]
+
+#? evaluate.Array() evaluate.Class() evaluate.Function() evaluate.Instance()
+evaluate.get_scopes_for_name()[0]

test/run.py

@@ -27,8 +27,8 @@ def run_completion_test(correct, source, line_nr, line, path):
         #profile.run('functions.complete("""%s""", %i, %i, "%s")'
         #                            % (source, line_nr, len(line), path))
     except Exception:
-        print('test @%s: %s' % (line_nr - 1, line))
         print(traceback.format_exc())
+        print('test @%s: %s' % (line_nr - 1, line))
         return 1
     else:
         # TODO remove set! duplicates should not be normal
@@ -63,8 +63,8 @@ def run_definition_test(correct, source, line_nr, line, correct_start, path):
             try:
                 should_be |= defs(line_nr - 1, start + correct_start)
             except Exception:
-                print('could not resolve %s indent %s' % (line_nr - 1, start))
                 print(traceback.format_exc())
+                print('could not resolve %s indent %s' % (line_nr - 1, start))
                 return 1
         # because the objects have different ids, `repr` it, then compare it.
         should_str = sorted(str(r) for r in should_be)
@@ -102,8 +102,8 @@ def run_goto_test(correct, source, line_nr, line, path):
     try:
         result = functions.goto(source, line_nr, index, path)
     except Exception:
-        print('test @%s: %s' % (line_nr - 1, line))
         print(traceback.format_exc())
+        print('test @%s: %s' % (line_nr - 1, line))
         return 1
     else:
         comp_str = str(sorted(r.description for r in result))