Separate the python syntax tree stuff from the non python stuff.

jedi/api/keywords.py

@@ -5,6 +5,7 @@ from jedi._compatibility import is_py3, is_py35
 from jedi import common
 from jedi.evaluate.filters import AbstractNameDefinition
 from jedi.parser.python.tree import Leaf
+
 try:
     from pydoc_data import topics as pydoc_topics
 except ImportError:

jedi/parser/python/parser.py

@@ -100,7 +100,7 @@ class Parser(object):
 
     def convert_node(self, grammar, type, children):
         """
-        Convert raw node information to a Node instance.
+        Convert raw node information to a PythonBaseNode instance.
 
         This is passed to the parser driver which calls it whenever a reduction of a
         grammar rule produces a new complete node, so that the tree is build
@@ -116,7 +116,7 @@ class Parser(object):
                 # ones and therefore have pseudo start/end positions and no
                 # prefixes. Just ignore them.
                 children = [children[0]] + children[2:-1]
-            return tree.Node(symbol, children)
+            return tree.PythonNode(symbol, children)
 
     def convert_leaf(self, grammar, type, value, prefix, start_pos):
         # print('leaf', repr(value), token.tok_name[type])
@@ -226,8 +226,8 @@ class ParserWithRecovery(Parser):
                     # suites without an indent in them get discarded.
                     break
                 elif symbol == 'simple_stmt' and len(nodes) > 1:
-                    # simple_stmt can just be turned into a Node, if there are
+                    # simple_stmt can just be turned into a PythonNode, if
-                    # enough statements. Ignore the rest after that.
+                    # there are enough statements. Ignore the rest after that.
                     break
             return index, symbol, nodes
 
@@ -236,7 +236,7 @@ class ParserWithRecovery(Parser):
             index -= 2
             (_, _, (type_, suite_nodes)) = stack[index]
             symbol = grammar.number2symbol[type_]
-            suite_nodes.append(tree.Node(symbol, list(nodes)))
+            suite_nodes.append(tree.PythonNode(symbol, list(nodes)))
             # Remove
             nodes[:] = []
             nodes = suite_nodes
@@ -251,7 +251,7 @@ class ParserWithRecovery(Parser):
                 # Otherwise the parser will get into trouble and DEDENT too early.
                 self._omit_dedent_list.append(self._indent_counter)
             else:
-                error_leaf = tree.ErrorLeaf(tok_name[typ].lower(), value, start_pos, prefix)
+                error_leaf = tree.PythonErrorLeaf(tok_name[typ].lower(), value, start_pos, prefix)
                 stack[-1][2][1].append(error_leaf)
 
     def _stack_removal(self, grammar, stack, arcs, start_index, value, start_pos):
@@ -266,7 +266,7 @@ class ParserWithRecovery(Parser):
                 failed_stack.append((symbol, nodes))
                 all_nodes += nodes
         if failed_stack:
-            stack[start_index - 1][2][1].append(tree.ErrorNode(all_nodes))
+            stack[start_index - 1][2][1].append(tree.PythonErrorNode(all_nodes))
 
         stack[start_index:] = []
         return failed_stack

jedi/parser/python/tree.py

@@ -39,8 +39,10 @@ from itertools import chain
 import textwrap
 import abc
 
-from jedi._compatibility import (Python3Method, encoding, is_py3, utf8_repr,
+from jedi._compatibility import (Python3Method, is_py3, utf8_repr,
                                  literal_eval, unicode)
+from jedi.parser.tree import (Node, BaseNode, Leaf, LeafWithNewlines,
+                              ErrorNode, ErrorLeaf)
 
 
 def _safe_literal_eval(value):
@@ -109,23 +111,7 @@ class DocstringMixin(object):
         return ''
 
 
-class Base(object):
+class PythonMixin():
-    """
-    This is just here to have an isinstance check, which is also used on
-    evaluate classes. But since they have sometimes a special type of
-    delegation, it is important for those classes to override this method.
-
-    I know that there is a chance to do such things with __instancecheck__, but
-    since Python 2.5 doesn't support it, I decided to do it this way.
-    """
-    __slots__ = ()
-
-    def get_root_node(self):
-        scope = self
-        while scope.parent is not None:
-            scope = scope.parent
-        return scope
-
     def get_parent_scope(self, include_flows=False):
         """
         Returns the underlying scope.
@@ -145,7 +131,7 @@ class Base(object):
         scope = self
         while scope.parent is not None:
             parent = scope.parent
-            if isinstance(scope, (Node, Leaf)) and parent.type != 'simple_stmt':
+            if isinstance(scope, (PythonNode, PythonLeaf)) and parent.type != 'simple_stmt':
                 if scope.type == 'testlist_comp':
                     try:
                         if isinstance(scope.children[1], CompFor):
@@ -194,161 +180,63 @@ class Base(object):
     def nodes_to_execute(self, last_added=False):
         raise NotImplementedError()
 
-    def get_next_sibling(self):
+    @Python3Method
-        """
+    def name_for_position(self, position):
-        The node immediately following the invocant in their parent's children
+        for c in self.children:
-        list. If the invocant does not have a next sibling, it is None
+            if isinstance(c, Leaf):
-        """
+                if isinstance(c, Name) and c.start_pos <= position <= c.end_pos:
-        # Can't use index(); we need to test by identity
+                    return c
-        for i, child in enumerate(self.parent.children):
+            else:
-            if child is self:
+                result = c.name_for_position(position)
+                if result is not None:
+                    return result
+        return None
+
+    @Python3Method
+    def get_statement_for_position(self, pos):
+        for c in self.children:
+            if c.start_pos <= pos <= c.end_pos:
+                if c.type not in ('decorated', 'simple_stmt', 'suite') \
+                        and not isinstance(c, (Flow, ClassOrFunc)):
+                    return c
+                else:
                     try:
-                    return self.parent.children[i + 1]
+                        return c.get_statement_for_position(pos)
-                except IndexError:
+                    except AttributeError:
+                        pass  # Must be a non-scope
         return None
 
-    def get_previous_sibling(self):
-        """
-        The node/leaf immediately preceding the invocant in their parent's
-        children list. If the invocant does not have a previous sibling, it is
-        None.
-        """
-        # Can't use index(); we need to test by identity
-        for i, child in enumerate(self.parent.children):
-            if child is self:
-                if i == 0:
-                    return None
-                return self.parent.children[i - 1]
 
-    def get_previous_leaf(self):
+class PythonLeaf(Leaf, PythonMixin):
-        """
+    pass
-        Returns the previous leaf in the parser tree.
-        Raises an IndexError if it's the first element.
-        """
-        node = self
-        while True:
-            c = node.parent.children
-            i = c.index(node)
-            if i == 0:
-                node = node.parent
-                if node.parent is None:
-                    return None
-            else:
-                node = c[i - 1]
-                break
-
-        while True:
-            try:
-                node = node.children[-1]
-            except AttributeError:  # A Leaf doesn't have children.
-                return node
-
-    def get_next_leaf(self):
-        """
-        Returns the previous leaf in the parser tree.
-        Raises an IndexError if it's the last element.
-        """
-        node = self
-        while True:
-            c = node.parent.children
-            i = c.index(node)
-            if i == len(c) - 1:
-                node = node.parent
-                if node.parent is None:
-                    return None
-            else:
-                node = c[i + 1]
-                break
-
-        while True:
-            try:
-                node = node.children[0]
-            except AttributeError:  # A Leaf doesn't have children.
-                return node
 
 
-class Leaf(Base):
+class PythonLeafWithNewlines(LeafWithNewlines, PythonMixin):
-    __slots__ = ('value', 'parent', 'line', 'indent', 'prefix')
+    pass
-
-    def __init__(self, value, start_pos, prefix=''):
-        self.value = value
-        self.start_pos = start_pos
-        self.prefix = prefix
-        self.parent = None
-
-    @property
-    def start_pos(self):
-        return self.line, self.indent
-
-    @start_pos.setter
-    def start_pos(self, value):
-        self.line = value[0]
-        self.indent = value[1]
-
-    def get_start_pos_of_prefix(self):
-        previous_leaf = self.get_previous_leaf()
-        if previous_leaf is None:
-            return self.line - self.prefix.count('\n'), 0  # It's the first leaf.
-        return previous_leaf.end_pos
-
-    @property
-    def end_pos(self):
-        return self.line, self.indent + len(self.value)
-
-    def move(self, line_offset):
-        self.line += line_offset
-
-    def get_first_leaf(self):
-        return self
-
-    def get_last_leaf(self):
-        return self
-
-    def get_code(self, normalized=False, include_prefix=True):
-        if normalized:
-            return self.value
-        if include_prefix:
-            return self.prefix + self.value
-        else:
-            return self.value
-
-    def nodes_to_execute(self, last_added=False):
-        return []
-
-    @utf8_repr
-    def __repr__(self):
-        return "<%s: %s start=%s>" % (type(self).__name__, self.value, self.start_pos)
 
 
-class LeafWithNewLines(Leaf):
+class PythonBaseNode(BaseNode, PythonMixin):
-    __slots__ = ()
+    pass
-
-    @property
-    def end_pos(self):
-        """
-        Literals and whitespace end_pos are more complicated than normal
-        end_pos, because the containing newlines may change the indexes.
-        """
-        lines = self.value.split('\n')
-        end_pos_line = self.line + len(lines) - 1
-        # Check for multiline token
-        if self.line == end_pos_line:
-            end_pos_indent = self.indent + len(lines[-1])
-        else:
-            end_pos_indent = len(lines[-1])
-        return end_pos_line, end_pos_indent
-
-    @utf8_repr
-    def __repr__(self):
-        return "<%s: %r>" % (type(self).__name__, self.value)
 
 
-class EndMarker(Leaf):
+class PythonErrorNode(ErrorNode, PythonMixin):
+    pass
+
+
+class PythonErrorLeaf(ErrorLeaf, PythonMixin):
+    pass
+
+
+class PythonNode(Node, PythonMixin):
+    pass
+
+
+class EndMarker(PythonLeaf):
     __slots__ = ()
     type = 'endmarker'
 
 
-class Newline(LeafWithNewLines):
+class Newline(PythonLeafWithNewlines):
     """Contains NEWLINE and ENDMARKER tokens."""
     __slots__ = ()
     type = 'newline'
@@ -358,7 +246,7 @@ class Newline(LeafWithNewLines):
         return "<%s: %s>" % (type(self).__name__, repr(self.value))
 
 
-class Name(Leaf):
+class Name(PythonLeaf):
     """
     A string. Sometimes it is important to know if the string belongs to a name
     or not.
@@ -398,7 +286,7 @@ class Name(Leaf):
             yield self
 
 
-class Literal(LeafWithNewLines):
+class Literal(PythonLeafWithNewlines):
     __slots__ = ()
 
     def eval(self):
@@ -415,7 +303,7 @@ class String(Literal):
     __slots__ = ()
 
 
-class Operator(Leaf):
+class Operator(PythonLeaf):
     type = 'operator'
     __slots__ = ()
 
@@ -440,7 +328,7 @@ class Operator(Leaf):
         return hash(self.value)
 
 
-class Keyword(Leaf):
+class Keyword(PythonLeaf):
     type = 'keyword'
     __slots__ = ()
 
@@ -461,215 +349,7 @@ class Keyword(Leaf):
         return hash(self.value)
 
 
-class BaseNode(Base):
+class Scope(PythonBaseNode, DocstringMixin):
-    """
-    The super class for Scope, Import, Name and Statement. Every object in
-    the parser tree inherits from this class.
-    """
-    __slots__ = ('children', 'parent')
-    type = None
-
-    def __init__(self, children):
-        """
-        Initialize :class:`BaseNode`.
-
-        :param children: The module in which this Python object locates.
-        """
-        for c in children:
-            c.parent = self
-        self.children = children
-        self.parent = None
-
-    def move(self, line_offset):
-        """
-        Move the Node's start_pos.
-        """
-        for c in self.children:
-            c.move(line_offset)
-
-    @property
-    def start_pos(self):
-        return self.children[0].start_pos
-
-    def get_start_pos_of_prefix(self):
-        return self.children[0].get_start_pos_of_prefix()
-
-    @property
-    def end_pos(self):
-        return self.children[-1].end_pos
-
-    def _get_code_for_children(self, children, normalized, include_prefix):
-        # TODO implement normalized (depending on context).
-        if include_prefix:
-            return "".join(c.get_code(normalized) for c in children)
-        else:
-            first = children[0].get_code(include_prefix=False)
-            return first + "".join(c.get_code(normalized) for c in children[1:])
-
-    def get_code(self, normalized=False, include_prefix=True):
-        return self._get_code_for_children(self.children, normalized, include_prefix)
-
-    @Python3Method
-    def name_for_position(self, position):
-        for c in self.children:
-            if isinstance(c, Leaf):
-                if isinstance(c, Name) and c.start_pos <= position <= c.end_pos:
-                    return c
-            else:
-                result = c.name_for_position(position)
-                if result is not None:
-                    return result
-        return None
-
-    def get_leaf_for_position(self, position, include_prefixes=False):
-        def binary_search(lower, upper):
-            if lower == upper:
-                element = self.children[lower]
-                if not include_prefixes and position < element.start_pos:
-                    # We're on a prefix.
-                    return None
-                # In case we have prefixes, a leaf always matches
-                try:
-                    return element.get_leaf_for_position(position, include_prefixes)
-                except AttributeError:
-                    return element
-
-
-            index = int((lower + upper) / 2)
-            element = self.children[index]
-            if position <= element.end_pos:
-                return binary_search(lower, index)
-            else:
-                return binary_search(index + 1, upper)
-
-        if not ((1, 0) <= position <= self.children[-1].end_pos):
-            raise ValueError('Please provide a position that exists within this node.')
-        return binary_search(0, len(self.children) - 1)
-
-    @Python3Method
-    def get_statement_for_position(self, pos):
-        for c in self.children:
-            if c.start_pos <= pos <= c.end_pos:
-                if c.type not in ('decorated', 'simple_stmt', 'suite') \
-                        and not isinstance(c, (Flow, ClassOrFunc)):
-                    return c
-                else:
-                    try:
-                        return c.get_statement_for_position(pos)
-                    except AttributeError:
-                        pass  # Must be a non-scope
-        return None
-
-    def get_first_leaf(self):
-        return self.children[0].get_first_leaf()
-
-    def get_last_leaf(self):
-        return self.children[-1].get_last_leaf()
-
-    def get_following_comment_same_line(self):
-        """
-        returns (as string) any comment that appears on the same line,
-        after the node, including the #
-        """
-        try:
-            if self.type == 'for_stmt':
-                whitespace = self.children[5].get_first_leaf().prefix
-            elif self.type == 'with_stmt':
-                whitespace = self.children[3].get_first_leaf().prefix
-            else:
-                whitespace = self.get_last_leaf().get_next_leaf().prefix
-        except AttributeError:
-            return None
-        except ValueError:
-            # TODO in some particular cases, the tree doesn't seem to be linked
-            # correctly
-            return None
-        if "#" not in whitespace:
-            return None
-        comment = whitespace[whitespace.index("#"):]
-        if "\r" in comment:
-            comment = comment[:comment.index("\r")]
-        if "\n" in comment:
-            comment = comment[:comment.index("\n")]
-        return comment
-
-    @utf8_repr
-    def __repr__(self):
-        code = self.get_code().replace('\n', ' ').strip()
-        if not is_py3:
-            code = code.encode(encoding, 'replace')
-        return "<%s: %s@%s,%s>" % \
-            (type(self).__name__, code, self.start_pos[0], self.start_pos[1])
-
-
-class Node(BaseNode):
-    """Concrete implementation for interior nodes."""
-    __slots__ = ('type',)
-
-    _IGNORE_EXECUTE_NODES = set([
-        'suite', 'subscriptlist', 'subscript', 'simple_stmt', 'sliceop',
-        'testlist_comp', 'dictorsetmaker', 'trailer', 'decorators',
-        'decorated', 'arglist', 'argument', 'exprlist', 'testlist',
-        'testlist_safe', 'testlist1'
-    ])
-
-    def __init__(self, type, children):
-        """
-        Initializer.
-
-        Takes a type constant (a symbol number >= 256), a sequence of
-        child nodes, and an optional context keyword argument.
-
-        As a side effect, the parent pointers of the children are updated.
-        """
-        super(Node, self).__init__(children)
-        self.type = type
-
-    def nodes_to_execute(self, last_added=False):
-        """
-        For static analysis.
-        """
-        result = []
-        if self.type not in Node._IGNORE_EXECUTE_NODES and not last_added:
-            result.append(self)
-            last_added = True
-
-        for child in self.children:
-            result += child.nodes_to_execute(last_added)
-        return result
-
-    def __repr__(self):
-        return "%s(%s, %r)" % (self.__class__.__name__, self.type, self.children)
-
-
-class ErrorNode(BaseNode):
-    """
-    TODO doc
-    """
-    __slots__ = ()
-    type = 'error_node'
-
-    def nodes_to_execute(self, last_added=False):
-        return []
-
-
-class ErrorLeaf(LeafWithNewLines):
-    """
-    TODO doc
-    """
-    __slots__ = ('original_type')
-    type = 'error_leaf'
-
-    def __init__(self, original_type, value, start_pos, prefix=''):
-        super(ErrorLeaf, self).__init__(value, start_pos, prefix)
-        self.original_type = original_type
-
-    def __repr__(self):
-        return "<%s: %s:%s, %s)>" % \
-            (type(self).__name__, self.original_type, repr(self.value), self.start_pos)
-
-
-class Scope(BaseNode, DocstringMixin):
     """
     Super class for the parser tree, which represents the state of a python
     text file.
@@ -811,7 +491,7 @@ class Module(Scope):
         return result
 
 
-class Decorator(BaseNode):
+class Decorator(PythonBaseNode):
     type = 'decorator'
     __slots__ = ()
 
@@ -1097,7 +777,7 @@ class Lambda(Function):
         return "<%s@%s>" % (self.__class__.__name__, self.start_pos)
 
 
-class Flow(BaseNode):
+class Flow(PythonBaseNode):
     __slots__ = ()
     FLOW_KEYWORDS = (
         'try', 'except', 'finally', 'else', 'if', 'elif', 'with', 'for', 'while'
@@ -1249,7 +929,7 @@ class WithStmt(Flow):
         return result
 
 
-class Import(BaseNode):
+class Import(PythonBaseNode):
     __slots__ = ()
 
     def path_for_name(self, name):
@@ -1399,7 +1079,7 @@ class ImportName(Import):
                     if alias is not None)
 
 
-class KeywordStatement(BaseNode):
+class KeywordStatement(PythonBaseNode):
     """
     For the following statements: `assert`, `del`, `global`, `nonlocal`,
     `raise`, `return`, `yield`, `return`, `yield`.
@@ -1456,7 +1136,7 @@ class ReturnStmt(KeywordStatement):
     __slots__ = ()
 
 
-class YieldExpr(BaseNode):
+class YieldExpr(PythonBaseNode):
     __slots__ = ()
 
     @property
@@ -1491,7 +1171,7 @@ def _defined_names(current):
     return names
 
 
-class ExprStmt(BaseNode, DocstringMixin):
+class ExprStmt(PythonBaseNode, DocstringMixin):
     type = 'expr_stmt'
     __slots__ = ()
 
@@ -1527,7 +1207,7 @@ class ExprStmt(BaseNode, DocstringMixin):
         return result
 
 
-class Param(BaseNode):
+class Param(PythonBaseNode):
     """
     It's a helper class that makes business logic with params much easier. The
     Python grammar defines no ``param`` node. It defines it in a different way
@@ -1597,7 +1277,7 @@ class Param(BaseNode):
         return self._get_code_for_children(children, False, False)
 
 
-class CompFor(BaseNode):
+class CompFor(PythonBaseNode):
     type = 'comp_for'
     __slots__ = ()
 

jedi/parser/tree.py

@@ -0,0 +1,353 @@
+from jedi._compatibility import utf8_repr, encoding, is_py3
+
+
+class _NodeOrLeaf(object):
+    """
+    This is just here to have an isinstance check, which is also used on
+    evaluate classes. But since they have sometimes a special type of
+    delegation, it is important for those classes to override this method.
+
+    I know that there is a chance to do such things with __instancecheck__, but
+    since Python 2.5 doesn't support it, I decided to do it this way.
+    """
+    __slots__ = ()
+
+    def get_root_node(self):
+        scope = self
+        while scope.parent is not None:
+            scope = scope.parent
+        return scope
+
+    def get_next_sibling(self):
+        """
+        The node immediately following the invocant in their parent's children
+        list. If the invocant does not have a next sibling, it is None
+        """
+        # Can't use index(); we need to test by identity
+        for i, child in enumerate(self.parent.children):
+            if child is self:
+                try:
+                    return self.parent.children[i + 1]
+                except IndexError:
+                    return None
+
+    def get_previous_sibling(self):
+        """
+        The node/leaf immediately preceding the invocant in their parent's
+        children list. If the invocant does not have a previous sibling, it is
+        None.
+        """
+        # Can't use index(); we need to test by identity
+        for i, child in enumerate(self.parent.children):
+            if child is self:
+                if i == 0:
+                    return None
+                return self.parent.children[i - 1]
+
+    def get_previous_leaf(self):
+        """
+        Returns the previous leaf in the parser tree.
+        Raises an IndexError if it's the first element.
+        """
+        node = self
+        while True:
+            c = node.parent.children
+            i = c.index(node)
+            if i == 0:
+                node = node.parent
+                if node.parent is None:
+                    return None
+            else:
+                node = c[i - 1]
+                break
+
+        while True:
+            try:
+                node = node.children[-1]
+            except AttributeError:  # A Leaf doesn't have children.
+                return node
+
+    def get_next_leaf(self):
+        """
+        Returns the previous leaf in the parser tree.
+        Raises an IndexError if it's the last element.
+        """
+        node = self
+        while True:
+            c = node.parent.children
+            i = c.index(node)
+            if i == len(c) - 1:
+                node = node.parent
+                if node.parent is None:
+                    return None
+            else:
+                node = c[i + 1]
+                break
+
+        while True:
+            try:
+                node = node.children[0]
+            except AttributeError:  # A Leaf doesn't have children.
+                return node
+
+
+class Leaf(_NodeOrLeaf):
+    __slots__ = ('value', 'parent', 'line', 'indent', 'prefix')
+
+    def __init__(self, value, start_pos, prefix=''):
+        self.value = value
+        self.start_pos = start_pos
+        self.prefix = prefix
+        self.parent = None
+
+    @property
+    def start_pos(self):
+        return self.line, self.indent
+
+    @start_pos.setter
+    def start_pos(self, value):
+        self.line = value[0]
+        self.indent = value[1]
+
+    def get_start_pos_of_prefix(self):
+        previous_leaf = self.get_previous_leaf()
+        if previous_leaf is None:
+            return self.line - self.prefix.count('\n'), 0  # It's the first leaf.
+        return previous_leaf.end_pos
+
+    @property
+    def end_pos(self):
+        return self.line, self.indent + len(self.value)
+
+    def move(self, line_offset):
+        self.line += line_offset
+
+    def get_first_leaf(self):
+        return self
+
+    def get_last_leaf(self):
+        return self
+
+    def get_code(self, normalized=False, include_prefix=True):
+        if normalized:
+            return self.value
+        if include_prefix:
+            return self.prefix + self.value
+        else:
+            return self.value
+
+    def nodes_to_execute(self, last_added=False):
+        return []
+
+    @utf8_repr
+    def __repr__(self):
+        return "<%s: %s start=%s>" % (type(self).__name__, self.value, self.start_pos)
+
+
+class LeafWithNewlines(Leaf):
+    __slots__ = ()
+
+    @property
+    def end_pos(self):
+        """
+        Literals and whitespace end_pos are more complicated than normal
+        end_pos, because the containing newlines may change the indexes.
+        """
+        lines = self.value.split('\n')
+        end_pos_line = self.line + len(lines) - 1
+        # Check for multiline token
+        if self.line == end_pos_line:
+            end_pos_indent = self.indent + len(lines[-1])
+        else:
+            end_pos_indent = len(lines[-1])
+        return end_pos_line, end_pos_indent
+
+    @utf8_repr
+    def __repr__(self):
+        return "<%s: %r>" % (type(self).__name__, self.value)
+
+
+class BaseNode(_NodeOrLeaf):
+    """
+    The super class for all nodes.
+
+    If you create custom nodes, you will probably want to inherit from this
+    ``BaseNode``.
+    """
+    __slots__ = ('children', 'parent')
+    type = None
+
+    def __init__(self, children):
+        """
+        Initialize :class:`BaseNode`.
+
+        :param children: The module in which this Python object locates.
+        """
+        for c in children:
+            c.parent = self
+        self.children = children
+        self.parent = None
+
+    def move(self, line_offset):
+        """
+        Move the Node's start_pos.
+        """
+        for c in self.children:
+            c.move(line_offset)
+
+    @property
+    def start_pos(self):
+        return self.children[0].start_pos
+
+    def get_start_pos_of_prefix(self):
+        return self.children[0].get_start_pos_of_prefix()
+
+    @property
+    def end_pos(self):
+        return self.children[-1].end_pos
+
+    def _get_code_for_children(self, children, normalized, include_prefix):
+        # TODO implement normalized (depending on context).
+        if include_prefix:
+            return "".join(c.get_code(normalized) for c in children)
+        else:
+            first = children[0].get_code(include_prefix=False)
+            return first + "".join(c.get_code(normalized) for c in children[1:])
+
+    def get_code(self, normalized=False, include_prefix=True):
+        return self._get_code_for_children(self.children, normalized, include_prefix)
+
+    def get_leaf_for_position(self, position, include_prefixes=False):
+        def binary_search(lower, upper):
+            if lower == upper:
+                element = self.children[lower]
+                if not include_prefixes and position < element.start_pos:
+                    # We're on a prefix.
+                    return None
+                # In case we have prefixes, a leaf always matches
+                try:
+                    return element.get_leaf_for_position(position, include_prefixes)
+                except AttributeError:
+                    return element
+
+
+            index = int((lower + upper) / 2)
+            element = self.children[index]
+            if position <= element.end_pos:
+                return binary_search(lower, index)
+            else:
+                return binary_search(index + 1, upper)
+
+        if not ((1, 0) <= position <= self.children[-1].end_pos):
+            raise ValueError('Please provide a position that exists within this node.')
+        return binary_search(0, len(self.children) - 1)
+
+    def get_first_leaf(self):
+        return self.children[0].get_first_leaf()
+
+    def get_last_leaf(self):
+        return self.children[-1].get_last_leaf()
+
+    def get_following_comment_same_line(self):
+        """
+        returns (as string) any comment that appears on the same line,
+        after the node, including the #
+        """
+        try:
+            if self.type == 'for_stmt':
+                whitespace = self.children[5].get_first_leaf().prefix
+            elif self.type == 'with_stmt':
+                whitespace = self.children[3].get_first_leaf().prefix
+            else:
+                whitespace = self.get_last_leaf().get_next_leaf().prefix
+        except AttributeError:
+            return None
+        except ValueError:
+            # TODO in some particular cases, the tree doesn't seem to be linked
+            # correctly
+            return None
+        if "#" not in whitespace:
+            return None
+        comment = whitespace[whitespace.index("#"):]
+        if "\r" in comment:
+            comment = comment[:comment.index("\r")]
+        if "\n" in comment:
+            comment = comment[:comment.index("\n")]
+        return comment
+
+    @utf8_repr
+    def __repr__(self):
+        code = self.get_code().replace('\n', ' ').strip()
+        if not is_py3:
+            code = code.encode(encoding, 'replace')
+        return "<%s: %s@%s,%s>" % \
+            (type(self).__name__, code, self.start_pos[0], self.start_pos[1])
+
+
+class Node(BaseNode):
+    """Concrete implementation for interior nodes."""
+    __slots__ = ('type',)
+
+    _IGNORE_EXECUTE_NODES = set([
+        'suite', 'subscriptlist', 'subscript', 'simple_stmt', 'sliceop',
+        'testlist_comp', 'dictorsetmaker', 'trailer', 'decorators',
+        'decorated', 'arglist', 'argument', 'exprlist', 'testlist',
+        'testlist_safe', 'testlist1'
+    ])
+
+    def __init__(self, type, children):
+        """
+        Initializer.
+
+        Takes a type constant (a symbol number >= 256), a sequence of
+        child nodes, and an optional context keyword argument.
+
+        As a side effect, the parent pointers of the children are updated.
+        """
+        super(Node, self).__init__(children)
+        self.type = type
+
+    def nodes_to_execute(self, last_added=False):
+        """
+        For static analysis.
+        """
+        result = []
+        if self.type not in Node._IGNORE_EXECUTE_NODES and not last_added:
+            result.append(self)
+            last_added = True
+
+        for child in self.children:
+            result += child.nodes_to_execute(last_added)
+        return result
+
+    def __repr__(self):
+        return "%s(%s, %r)" % (self.__class__.__name__, self.type, self.children)
+
+
+class ErrorNode(BaseNode):
+    """
+    TODO doc
+    """
+    __slots__ = ()
+    type = 'error_node'
+
+    def nodes_to_execute(self, last_added=False):
+        return []
+
+
+class ErrorLeaf(LeafWithNewlines):
+    """
+    TODO doc
+    """
+    __slots__ = ('original_type')
+    type = 'error_leaf'
+
+    def __init__(self, original_type, value, start_pos, prefix=''):
+        super(ErrorLeaf, self).__init__(value, start_pos, prefix)
+        self.original_type = original_type
+
+    def __repr__(self):
+        return "<%s: %s:%s, %s)>" % \
+            (type(self).__name__, self.original_type, repr(self.value), self.start_pos)
+
+