Remove nodes_to_execute in favor of a function in parser_utils.

jedi/api/__init__.py

@@ -16,6 +16,7 @@ import sys
 from jedi.parser.python import load_grammar
 from jedi.parser.python import tree
 from jedi.parser.python import parse
+from jedi.parser_utils import get_executable_nodes
 from jedi import debug
 from jedi import settings
 from jedi import common
@@ -331,7 +332,7 @@ class Script(object):
         module_node = self._get_module_node()
         self._evaluator.analysis_modules = [module_node]
         try:
-            for node in module_node.nodes_to_execute():
+            for node in get_executable_nodes(module_node):
                 context = self._get_module().create_context(node)
                 if node.type in ('funcdef', 'classdef'):
                     # TODO This is stupid, should be private

jedi/evaluate/__init__.py

@@ -450,7 +450,8 @@ class Evaluator(object):
                 raise NotImplementedError
             if def_.type == 'expr_stmt' and name in def_.get_defined_names():
                 return self.eval_statement(context, def_, name)
-            elif def_.type == 'for_stmt':
+            elif def_.type == 'for_stmt' and \
+                    name.start_pos < def_.children[1].end_pos:
                 container_types = self.eval_element(context, def_.children[3])
                 cn = ContextualizedNode(context, def_.children[3])
                 for_types = iterable.py__iter__types(self, container_types, cn)

jedi/parser/python/tree.py

@@ -25,16 +25,11 @@ Any subclasses of :class:`Scope`, including :class:`Module` has an attribute
 [<ImportName: import os@1,0>]
 
 See also :attr:`Scope.subscopes` and :attr:`Scope.statements`.
-
-For static analysis purposes there exists a method called
-``nodes_to_execute`` on all nodes and leaves. It's documented in the static
-anaylsis documentation.
 """
 
 from inspect import cleandoc
 from itertools import chain
 import textwrap
-import abc
 
 from jedi._compatibility import (Python3Method, is_py3, utf8_repr,
                                  literal_eval, unicode)
@@ -143,10 +138,6 @@ class PythonMixin():
         # Default is not being a scope. Just inherit from Scope.
         return False
 
-    @abc.abstractmethod
-    def nodes_to_execute(self, last_added=False):
-        raise NotImplementedError()
-
     @Python3Method
     def name_for_position(self, position):
         for c in self.children:
@@ -256,10 +247,6 @@ class Name(_LeafWithoutNewlines):
                                  'comp_for', 'with_stmt') \
                 and self in stmt.get_defined_names()
 
-    def nodes_to_execute(self, last_added=False):
-        if last_added is False:
-            yield self
-
 
 class Literal(PythonLeaf):
     __slots__ = ()
@@ -431,13 +418,6 @@ class Module(Scope):
                         return True
         return False
 
-    def nodes_to_execute(self, last_added=False):
-        # Yield itself, class needs to be executed for decorator checks.
-        result = []
-        for child in self.children:
-            result += child.nodes_to_execute()
-        return result
-
     @property
     def used_names(self):
         if self._used_names is None:
@@ -464,13 +444,6 @@ class Decorator(PythonBaseNode):
     type = 'decorator'
     __slots__ = ()
 
-    def nodes_to_execute(self, last_added=False):
-        if self.children[-2] == ')':
-            node = self.children[-3]
-            if node != '(':
-                return node.nodes_to_execute()
-        return []
-
 
 class ClassOrFunc(Scope):
     __slots__ = ()
@@ -528,34 +501,6 @@ class Class(ClassOrFunc):
                     sub.get_call_signature(func_name=self.name), docstr)
         return docstr
 
-    def nodes_to_execute(self, last_added=False):
-        # Yield itself, class needs to be executed for decorator checks.
-        yield self
-        # Super arguments.
-        arglist = self.get_super_arglist()
-        try:
-            children = arglist.children
-        except AttributeError:
-            if arglist is not None:
-                for node_to_execute in arglist.nodes_to_execute():
-                    yield node_to_execute
-        else:
-            for argument in children:
-                if argument.type == 'argument':
-                    # metaclass= or list comprehension or */**
-                    raise NotImplementedError('Metaclasses not implemented')
-                else:
-                    for node_to_execute in argument.nodes_to_execute():
-                        yield node_to_execute
-
-        # care for the class suite:
-        for node in self.children[self.children.index(':'):]:
-            # This could be easier without the fast parser. But we need to find
-            # the position of the colon, because everything after it can be a
-            # part of the class, not just its suite.
-            for node_to_execute in node.nodes_to_execute():
-                yield node_to_execute
-
 
 def _create_params(parent, argslist_list):
     """
@@ -676,21 +621,6 @@ class Function(ClassOrFunc):
         docstr = self.raw_doc
         return '%s\n\n%s' % (self.get_call_signature(), docstr)
 
-    def nodes_to_execute(self, last_added=False):
-        # Yield itself, functions needs to be executed for decorator checks.
-        yield self
-        for param in self.params:
-            if param.default is not None:
-                yield param.default
-        # care for the function suite:
-        for node in self.children[4:]:
-            # This could be easier without the fast parser. The fast parser
-            # allows that the 4th position is empty or that there's even a
-            # fifth element (another function/class). So just scan everything
-            # after colon.
-            for node_to_execute in node.nodes_to_execute():
-                yield node_to_execute
-
 
 class Lambda(Function):
     """
@@ -734,14 +664,6 @@ class Lambda(Function):
     def yields(self):
         return []
 
-    def nodes_to_execute(self, last_added=False):
-        for param in self.params:
-            if param.default is not None:
-                yield param.default
-        # Care for the lambda test (last child):
-        for node_to_execute in self.children[-1].nodes_to_execute():
-            yield node_to_execute
-
     def __repr__(self):
         return "<%s@%s>" % (self.__class__.__name__, self.start_pos)
 
@@ -752,11 +674,6 @@ class Flow(PythonBaseNode):
         'try', 'except', 'finally', 'else', 'if', 'elif', 'with', 'for', 'while'
     )
 
-    def nodes_to_execute(self, last_added=False):
-        for child in self.children:
-            for node_to_execute in child.nodes_to_execute():
-                yield node_to_execute
-
     def get_branch_keyword(self, node):
         start_pos = node.start_pos
         if not (self.start_pos < start_pos <= self.end_pos):
@@ -857,16 +774,6 @@ class TryStmt(Flow):
             elif node == 'except':
                 yield None
 
-    def nodes_to_execute(self, last_added=False):
-        result = []
-        for child in self.children[2::3]:
-            result += child.nodes_to_execute()
-        for child in self.children[0::3]:
-            if child.type == 'except_clause':
-                # Add the test node and ignore the `as NAME` definition.
-                result += child.children[1].nodes_to_execute()
-        return result
-
 
 class WithStmt(Flow):
     type = 'with_stmt'
@@ -887,16 +794,6 @@ class WithStmt(Flow):
             if node.type == 'with_item':
                 return node.children[0]
 
-    def nodes_to_execute(self, last_added=False):
-        result = []
-        for child in self.children[1::2]:
-            if child.type == 'with_item':
-                # Just ignore the `as EXPR` part - at least for now, because
-                # most times it's just a name.
-                child = child.children[0]
-            result += child.nodes_to_execute()
-        return result
-
 
 class Import(PythonBaseNode):
     __slots__ = ()
@@ -919,14 +816,6 @@ class Import(PythonBaseNode):
     def is_star_import(self):
         return self.children[-1] == '*'
 
-    def nodes_to_execute(self, last_added=False):
-        """
-        `nodes_to_execute` works a bit different for imports, because the names
-        itself cannot directly get resolved (except on itself).
-        """
-        # TODO couldn't we return the names? Would be nicer.
-        return [self]
-
 
 class ImportFrom(Import):
     type = 'import_from'
@@ -1070,12 +959,6 @@ class KeywordStatement(PythonBaseNode):
     def keyword(self):
         return self.children[0].value
 
-    def nodes_to_execute(self, last_added=False):
-        result = []
-        for child in self.children:
-            result += child.nodes_to_execute()
-        return result
-
 
 class AssertStmt(KeywordStatement):
     __slots__ = ()
@@ -1093,13 +976,6 @@ class GlobalStmt(KeywordStatement):
     def get_global_names(self):
         return self.children[1::2]
 
-    def nodes_to_execute(self, last_added=False):
-        """
-        The global keyword allows to define any name. Even if it doesn't
-        exist.
-        """
-        return []
-
 
 class ReturnStmt(KeywordStatement):
     __slots__ = ()
@@ -1112,12 +988,6 @@ class YieldExpr(PythonBaseNode):
     def type(self):
         return 'yield_expr'
 
-    def nodes_to_execute(self, last_added=False):
-        if len(self.children) > 1:
-            return self.children[1].nodes_to_execute()
-        else:
-            return []
-
 
 def _defined_names(current):
     """
@@ -1167,14 +1037,6 @@ class ExprStmt(PythonBaseNode, DocstringMixin):
         except IndexError:
             return None
 
-    def nodes_to_execute(self, last_added=False):
-        # I think evaluating the statement (and possibly returned arrays),
-        # should be enough for static analysis.
-        result = [self]
-        for child in self.children:
-            result += child.nodes_to_execute(last_added=True)
-        return result
-
 
 class Param(PythonBaseNode):
     """
@@ -1273,16 +1135,3 @@ class CompFor(PythonBaseNode):
 
     def get_defined_names(self):
         return _defined_names(self.children[1])
-
-    def nodes_to_execute(self, last_added=False):
-        last = self.children[-1]
-        if last.type == 'comp_if':
-            for node in last.children[-1].nodes_to_execute():
-                yield node
-            last = self.children[-2]
-        elif last.type == 'comp_for':
-            for node in last.nodes_to_execute():
-                yield node
-            last = self.children[-2]
-        for node in last.nodes_to_execute():
-            yield node

jedi/parser/tree.py

@@ -132,9 +132,6 @@ class Leaf(_NodeOrLeaf):
         else:
             return self.value
 
-    def nodes_to_execute(self, last_added=False):
-        return []
-
     @property
     def end_pos(self):
         """
@@ -276,13 +273,6 @@ class Node(BaseNode):
     """Concrete implementation for interior nodes."""
     __slots__ = ('type',)
 
-    _IGNORE_EXECUTE_NODES = set([
-        'suite', 'subscriptlist', 'subscript', 'simple_stmt', 'sliceop',
-        'testlist_comp', 'dictorsetmaker', 'trailer', 'decorators',
-        'decorated', 'arglist', 'argument', 'exprlist', 'testlist',
-        'testlist_safe', 'testlist1'
-    ])
-
     def __init__(self, type, children):
         """
         Initializer.
@@ -295,19 +285,6 @@ class Node(BaseNode):
         super(Node, self).__init__(children)
         self.type = type
 
-    def nodes_to_execute(self, last_added=False):
-        """
-        For static analysis.
-        """
-        result = []
-        if self.type not in Node._IGNORE_EXECUTE_NODES and not last_added:
-            result.append(self)
-            last_added = True
-
-        for child in self.children:
-            result += child.nodes_to_execute(last_added)
-        return result
-
     def __repr__(self):
         return "%s(%s, %r)" % (self.__class__.__name__, self.type, self.children)
 
@@ -319,9 +296,6 @@ class ErrorNode(BaseNode):
     __slots__ = ()
     type = 'error_node'
 
-    def nodes_to_execute(self, last_added=False):
-        return []
-
 
 class ErrorLeaf(Leaf):
     """
@@ -337,5 +311,3 @@ class ErrorLeaf(Leaf):
     def __repr__(self):
         return "<%s: %s:%s, %s)>" % \
             (type(self).__name__, self.original_type, repr(self.value), self.start_pos)
-
-

jedi/parser_utils.py

@@ -0,0 +1,76 @@
+from jedi.parser.python.tree import PythonLeaf
+_IGNORE_EXECUTE_NODES = set([
+    'suite', 'subscriptlist', 'subscript', 'simple_stmt', 'sliceop',
+    'testlist_comp', 'dictorsetmaker', 'trailer', 'decorators',
+    'decorated', 'arglist', 'argument', 'exprlist', 'testlist',
+    'testlist_safe', 'testlist1', 'global_stmt', 'file_input', 'for_stmt',
+    'while_stmt', 'if_stmt', 'try_stmt', 'with_stmt', 'comp_for', 'comp_if',
+    'param', 'except_clause', 'dotted_name', 'keyword_stmt', 'return_stmt',
+    'del_stmt', 'pass_stmt', 'nonlocal_stmt', 'assert_stmt', 'break_stmt',
+    'continue_stmt', 'raise_stmt', 'yield_stmt'
+])
+
+return_ = 'import_name', 'import_from'
+
+
+# last added: Flow, KeywordStatement
+
+
+def get_executable_nodes(node, last_added=False):
+    """
+    For static analysis.
+    """
+    result = []
+    typ = node.type
+    if typ == 'classdef':
+        # Yield itself, class needs to be executed for decorator checks.
+        result.append(node)
+        # Super arguments.
+        arglist = node.get_super_arglist()
+        try:
+            children = arglist.children
+        except AttributeError:
+            if arglist is not None:
+                result += get_executable_nodes(arglist)
+        else:
+            for argument in children:
+                if argument.type == 'argument':
+                    # metaclass= or list comprehension or */**
+                    raise NotImplementedError('Metaclasses not implemented, yet.')
+                else:
+                    result += get_executable_nodes(argument)
+
+        # Care for the class suite:
+        suite = node.children[-1]
+        result += get_executable_nodes(suite)
+    elif typ == 'yield_expr':
+        if len(node.children) > 1:
+            # TODO delete?
+            result += get_executable_nodes(node.children[1])
+    elif typ == 'name':
+        next_leaf = node.get_next_leaf()
+        if last_added is False and node.parent.type != 'param' and next_leaf != '=':
+            result.append(node)
+    elif typ == 'expr_stmt':
+        # I think evaluating the statement (and possibly returned arrays),
+        # should be enough for static analysis.
+        result.append(node)
+        for child in node.children:
+            result += get_executable_nodes(child, last_added=True)
+    elif isinstance(node, PythonLeaf):
+        pass
+    elif typ == 'decorator':
+        # decorator
+        if node.children[-2] == ')':
+            node = children[-3]
+            if node != '(':
+                result += get_executable_nodes(node)
+    else:
+        if node.type not in _IGNORE_EXECUTE_NODES and not last_added:
+            result.append(node)
+            #last_added = True
+
+        for child in node.children:
+            result += get_executable_nodes(child, last_added)
+
+    return result

test/static_analysis/comprehensions.py

@@ -17,7 +17,7 @@ tuple(a + 3 for a in [''])
 # Some variables within are not defined
 # ----------
 
-abcdef = None
+abcdef = []
 #! 12 name-error
 [1 for a in NOT_DEFINFED for b in abcdef if 1]