jedi/parsing.py

"""
Maintainer: David Halter <davidhalter88@gmail.com>
Version: 0.1

py_fuzzyparser parses python code, with the goal of a good representation of
the code within a tree structure. Variables, Classes and Functions are defined
within this tree structure, containing their exact locations in the code.
It is also a primary goal to work with code which contains syntax errors.

This behaviour may be used to refactor, modify, search and complete code.

To understand this code it is extremely important to understand the behaviour
of the python module 'tokenize'.

This original codebase of this parser, which has been refactored and heavily
changed, was programmed by Aaron Griffin <aaronmgriffin@gmail.com>.

**The structure of the following script:**
A Scope has
 - imports (Import)
 - subscopes (Scope, Class, Function, Flow)
 - statements (Statement)

All those classes are being generated by PyFuzzyParser, which takes python text
as input.

Ignored statements:
 - print (no use for it, just slows down)
 - exec (dangerous - not controllable)

TODO take special care for future imports
TODO check meta classes
"""
from _compatibility import next

import tokenize
import cStringIO
import re

import debug


class ParserError(Exception):
    pass


def indent_block(text, indention="    "):
    """ This function indents a text block with a default of four spaces """
    temp = ''
    while text and text[-1] == '\n':
        temp += text[-1]
        text = text[:-1]
    lines = text.split('\n')
    return '\n'.join(map(lambda s: indention + s, lines)) + temp


class Simple(object):
    """
    The super class for Scope, Import, Name and Statement. Every object in
    the parser tree inherits from this class.
    """
    def __init__(self, indent, line_nr, line_end=None):
        self.indent = indent
        self.line_nr = line_nr
        self.line_end = line_end
        self.parent = None

    def get_parent_until(self, *classes):
        """ Takes always the parent, until one class (not a Class) """
        scope = self
        while not (scope.parent is None or scope.__class__ in classes):
            scope = scope.parent
        return scope

    def __repr__(self):
        code = self.get_code().replace('\n', ' ')
        return "<%s: %s@%s>" % \
                (self.__class__.__name__, code, self.line_nr)


class Scope(Simple):
    """
    Super class for the parser tree, which represents the state of a python
    text file.
    A Scope manages and owns its subscopes, which are classes and functions, as
    well as variables and imports. It is used to access the structure of python
    files.

    :param indent: The indent level of the flow statement.
    :type indent: int
    :param line_nr: Line number of the flow statement.
    :type line_nr: int
    :param docstr: The docstring for the current Scope.
    :type docstr: str
    """
    def __init__(self, indent, line_nr, docstr=''):
        super(Scope, self).__init__(indent, line_nr)
        self.subscopes = []
        self.imports = []
        self.statements = []
        self.docstr = docstr

    def add_scope(self, sub, decorators):
        # print 'push scope: [%s@%s]' % (sub.line_nr, sub.indent)
        sub.parent = self
        sub.decorators = decorators
        self.subscopes.append(sub)
        return sub

    def add_statement(self, stmt):
        """
        Used to add a Statement or a Scope.
        A statement would be a normal command (Statement) or a Scope (Flow).
        """
        stmt.parent = self
        self.statements.append(stmt)
        return stmt

    def add_docstr(self, string):
        """ Clean up a docstring """

        # TODO use prefixes, to format the doc strings
        # scan for string prefixes like r, u, etc.
        index1 = string.find("'")
        index2 = string.find('"')
        index = index1 if index1 < index2 and index1 > -1 else index2
        prefix = string[:index]
        d = string[index:]
        debug.dbg('add_docstr', d, prefix)

        # now clean docstr
        d = d.replace('\n', ' ')
        d = d.replace('\t', ' ')
        while d.find('  ') > -1:
            d = d.replace('  ', ' ')
        while d[0] in '"\'\t ':
            d = d[1:]
        while d[-1] in '"\'\t ':
            d = d[:-1]
        debug.dbg("Scope(%s)::docstr = %s" % (self, d))
        self.docstr = d

    def add_import(self, imp):
        self.imports.append(imp)
        imp.parent = self

    def get_imports(self):
        """ Gets also the imports within flow statements """
        i = [] + self.imports
        for s in self.statements:
            if isinstance(s, Scope):
                i += s.get_imports()
        return i

    def get_code(self, first_indent=False, indention="    "):
        """
        :return: Returns the code of the current scope.
        :rtype: str
        """
        string = ""
        if len(self.docstr) > 0:
            string += '"""' + self.docstr + '"""\n'
        for i in self.imports:
            string += i.get_code()
        for sub in self.subscopes:
            #string += str(sub.line_nr)
            string += sub.get_code(first_indent=True, indention=indention)
        for stmt in self.statements:
            string += stmt.get_code()

        if first_indent:
            string = indent_block(string, indention=indention)
        return string

    def get_set_vars(self):
        """
        Get all the names, that are active and accessible in the current
        scope.

        :return: list of Name
        :rtype: list
        """
        n = []
        for stmt in self.statements:
            try:
                n += stmt.get_set_vars(True)
            except TypeError:
                n += stmt.get_set_vars()

        # function and class names
        n += [s.name for s in self.subscopes]

        for i in self.imports:
            if not i.star:
                n += i.get_defined_names()

        return n

    def get_defined_names(self):
        return [n for n in self.get_set_vars() \
                            if isinstance(n, Import) or len(n) == 1]

    def is_empty(self):
        """
        :return: True if there are no subscopes, imports and statements.
        :rtype: bool
        """
        return not (self.imports or self.subscopes or self.statements)

    def __repr__(self):
        try:
            name = self.name
        except AttributeError:
            try:
                name = self.command
            except AttributeError:
                name = self.module_path

        return "<%s: %s@%s-%s>" % \
                (self.__class__.__name__, name, self.line_nr, self.line_end)


class GlobalScope(Scope):
    """
    The top scope, which is a module.
    I don't know why I didn't name it Module :-)
    """
    def __init__(self, module_path, docstr=''):
        super(GlobalScope, self).__init__(module_path, docstr)
        self.module_path = module_path
        self.global_vars = []

    def add_global(self, name):
        """
        Global means in these context a function (subscope) which has a global
        statement.
        This is only relevant for the top scope.

        :param name: The name of the global.
        :type name: Name
        """
        self.global_vars.append(name)
        # set no parent here, because globals are not defined in this scope.

    def get_set_vars(self):
        n = super(GlobalScope, self).get_set_vars()
        n += self.global_vars
        return n


class Class(Scope):
    """
    Used to store the parsed contents of a python class.

    :param name: The Class name.
    :type name: string
    :param name: The super classes of a Class.
    :type name: list
    :param indent: The indent level of the flow statement.
    :type indent: int
    :param line_nr: Line number of the flow statement.
    :type line_nr: int
    :param docstr: The docstring for the current Scope.
    :type docstr: str
    """
    def __init__(self, name, supers, indent, line_nr, docstr=''):
        super(Class, self).__init__(indent, line_nr, docstr)
        self.name = name
        name.parent = self
        self.supers = supers
        for s in self.supers:
            s.parent = self
        self.decorators = []

    def get_code(self, first_indent=False, indention="    "):
        str = "\n".join('@' + stmt.get_code() for stmt in self.decorators)
        str += 'class %s' % (self.name)
        if len(self.supers) > 0:
            sup = ','.join(stmt.code for stmt in self.supers)
            str += '(%s)' % sup
        str += ':\n'
        str += super(Class, self).get_code(True, indention)
        if self.is_empty():
            str += "pass\n"
        return str


class Function(Scope):
    """
    Used to store the parsed contents of a python function.

    :param name: The Function name.
    :type name: string
    :param params: The parameters (Statement) of a Function.
    :type name: list
    :param indent: The indent level of the flow statement.
    :type indent: int
    :param line_nr: Line number of the flow statement.
    :type line_nr: int
    :param docstr: The docstring for the current Scope.
    :type docstr: str
    """
    def __init__(self, name, params, indent, line_nr, docstr=''):
        Scope.__init__(self, indent, line_nr, docstr)
        self.name = name
        name.parent = self
        self.params = params
        for p in params:
            p.parent = self
        self.decorators = []
        self.returns = []
        self.is_generator = False

        # callback to set the function
        self.param_cb = None

    def get_code(self, first_indent=False, indention="    "):
        str = "\n".join('@' + stmt.get_code() for stmt in self.decorators)
        params = ','.join([stmt.code for stmt in self.params])
        str += "def %s(%s):\n" % (self.name, params)
        str += super(Function, self).get_code(True, indention)
        if self.is_empty():
            str += "pass\n"
        return str

    def get_set_vars(self):
        n = super(Function, self).get_set_vars()
        if self.param_cb or False:
            # this is the really ugly part, where the functional style of this
            # get methods is broken, it executes a callback.
            # This is important, because something has to inject the params
            # into the functions, with the right values.
            n += self.param_cb()
        else:
            for p in self.params:
                try:
                    n.append(p.get_name())
                except IndexError:
                    debug.warning("multiple names in param %s" % n)
        return n


class Flow(Scope):
    """
    Used to describe programming structure - flow statements,
    which indent code, but are not classes or functions:

    - for
    - while
    - if
    - try
    - with

    Therefore statements like else, except and finally are also here,
    they are now saved in the root flow elements, but in the next variable.

    :param command: The flow command, if, while, else, etc.
    :type command: str
    :param inits: The initializations of a flow -> while 'statement'.
    :type inits: list(Statement)
    :param indent: The indent level of the flow statement.
    :type indent: int
    :param line_nr: Line number of the flow statement.
    :type line_nr: int
    :param set_vars: Local variables used in the for loop (only there).
    :type set_vars: list
    """
    def __init__(self, command, inits, indent, line_nr, set_vars=None):
        super(Flow, self).__init__(indent, line_nr, '')
        self.command = command
        # These have to be statements, because of with, which takes multiple.
        self.inits = inits
        for s in inits:
            s.parent = self
        if set_vars == None:
            self.set_vars = []
        else:
            self.set_vars = set_vars
            for s in self.set_vars:
                s.parent = self
        self.next = None

    def get_code(self, first_indent=False, indention="    "):
        if self.set_vars:
            vars = ",".join(map(lambda x: x.get_code(), self.set_vars))
            vars += ' in '
        else:
            vars = ''

        stmts = []
        for s in self.inits:
            stmts.append(s.get_code(new_line=False))
        stmt = ', '.join(stmts)
        str = "%s %s%s:\n" % (self.command, vars, stmt)
        str += super(Flow, self).get_code(True, indention)
        if self.next:
            str += self.next.get_code()
        return str

    def get_set_vars(self, is_internal_call=False):
        """
        Get the names for the flow. This includes also a call to the super
        class.
        :param is_internal_call: defines an option for internal files to crawl\
        through this class. Normally it will just call its superiors, to\
        generate the output.
        """
        if is_internal_call:
            n = list(self.set_vars)
            for s in self.inits:
                n += s.set_vars
            if self.next:
                n += self.next.get_set_vars(is_internal_call)
            n += super(Flow, self).get_set_vars()
            return n
        else:
            return self.get_parent_until(Class, Function).get_set_vars()

    def get_imports(self):
        i = super(Flow, self).get_imports()
        if self.next:
            i += self.next.get_imports()
        return i

    def set_next(self, next):
        """ Set the next element in the flow, those are else, except, etc. """
        if self.next:
            return self.next.set_next(next)
        else:
            self.next = next
            next.parent = self.parent
            return next


class ForFlow(Flow):
    """
    Used for the for loop, because there are two statement parts.
    """
    def __init__(self, command, inits, indent, line_nr, set_stmt):
        super(ForFlow, self).__init__(command, inits, indent, line_nr,
                                        set_stmt.used_vars)
        self.set_stmt = set_stmt


class Import(Simple):
    """
    Stores the imports of any Scopes.

    >>> 1+1
    2

    :param line_nr: Line number.
    :type line_nr: int
    :param namespace: The import, can be empty if a star is given
    :type namespace: Name
    :param alias: The alias of a namespace(valid in the current namespace).
    :type alias: Name
    :param from_ns: Like the namespace, can be equally used.
    :type from_ns: Name
    :param star: If a star is used -> from time import *.
    :type star: bool
    """
    def __init__(self, indent, line_nr, line_end, namespace, alias='', \
                    from_ns='', star=False, relative_count=None):
        super(Import, self).__init__(indent, line_nr, line_end)

        self.namespace = namespace
        if namespace:
            namespace.parent = self

        self.alias = alias
        if alias:
            alias.parent = self

        self.from_ns = from_ns
        if from_ns:
            from_ns.parent = self

        self.star = star
        self.relative_count = relative_count

    def get_code(self):
        if self.alias:
            ns_str = "%s as %s" % (self.namespace, self.alias)
        else:
            ns_str = str(self.namespace)
        if self.from_ns:
            if self.star:
                ns_str = '*'
            return "from %s import %s" % (self.from_ns, ns_str) + '\n'
        else:
            return "import " + ns_str + '\n'

    def get_defined_names(self):
        if self.star:
            return [self]
        return [self.alias] if self.alias else [self.namespace]


class Statement(Simple):
    """
    This is the class for all the possible statements. Which means, this class
    stores pretty much all the Python code, except functions, classes, imports,
    and flow functions like if, for, etc.

    :param code: The full code of a statement. This is import, if one wants \
    to execute the code at some level.
    :param code: str
    :param set_vars: The variables which are defined by the statement.
    :param set_vars: str
    :param used_funcs: The functions which are used by the statement.
    :param used_funcs: str
    :param used_vars: The variables which are used by the statement.
    :param used_vars: str
    :param token_list: Token list which is also peppered with Name.
    :param token_list: list
    :param indent: The indent level of the flow statement.
    :type indent: int
    :param line_nr: Line number of the flow statement.
    :type line_nr: int
    """
    def __init__(self, code, set_vars, used_funcs, used_vars, token_list,
            indent, line_nr, line_end):
        super(Statement, self).__init__(indent, line_nr, line_end)
        self.code = code
        self.set_vars = set_vars
        self.used_funcs = used_funcs
        self.used_vars = used_vars
        self.token_list = token_list
        for s in set_vars + used_funcs + used_vars:
            s.parent = self

        # cache
        self._assignment_calls = None
        self._assignment_details = None

    def get_code(self, new_line=True):
        if new_line:
            return self.code + '\n'
        else:
            return self.code

    def get_set_vars(self):
        """ Get the names for the statement. """
        return list(self.set_vars)

    @property
    def assignment_details(self):
        if self._assignment_details is None:
            # normally, this calls sets this variable
            self.get_assignment_calls()
        # it may not have been set by get_assignment_calls -> just use an empty
        # array
        return self._assignment_details or []

    def is_global(self):
        # first keyword of the first token is global -> must be a global
        return str(self.token_list[0]) == "global"

    def get_assignment_calls(self):
        """
        This is not done in the main parser, because it might be slow and
        most of the statements won't need this data anyway. This is something
        'like' a lazy execution.

        This is not really nice written, sorry for that. If you plan to replace
        it and make it nicer, that would be cool :-)
        """
        if self._assignment_calls:
            return self._assignment_calls
        self._assignment_details = []
        result = Array(Array.NOARRAY, self)
        top = result
        level = 0
        is_chain = False
        close_brackets = False

        debug.dbg('tok_list', self.token_list)
        tok_iter = enumerate(self.token_list)
        for i, tok_temp in tok_iter:
            #print 'tok', tok_temp, result
            try:
                token_type, tok, indent = tok_temp
            except TypeError:
                # the token is a Name, which has already been parsed
                tok = tok_temp
                token_type = None
            except ValueError:
                debug.warning("unkown value, shouldn't happen",
                                tok_temp, type(tok_temp))
                raise
            else:
                if tok in ['return', 'yield'] or level == 0 and \
                        '=' in tok and not tok in ['>=', '<=', '==', '!=']:
                    # This means, there is an assignment here.
                    # TODO there may be multiple assignments: a = b = 1

                    self._assignment_details.append((tok, top))
                    # All these calls wouldn't be important if nonlocal would
                    # exist. -> Initialize the first item again.
                    result = Array(Array.NOARRAY, self)
                    top = result
                    level = 0
                    close_brackets = False
                    is_chain = False
                    continue
                elif tok == 'as':
                    next(tok_iter)
                    continue

            brackets = {'(': Array.NOARRAY, '[': Array.LIST, '{': Array.SET}
            is_call = lambda: result.__class__ == Call
            is_call_or_close = lambda: is_call() or close_brackets

            if isinstance(tok, Name) or token_type in [tokenize.STRING,
                                                 tokenize.NUMBER]:  # names
                c_type = Call.NAME
                if token_type == tokenize.STRING:
                    c_type = Call.STRING
                elif token_type == tokenize.NUMBER:
                    c_type = Call.NUMBER

                if is_chain:
                    call = Call(tok, c_type, self, result)
                    result = result.set_next_chain_call(call)
                    is_chain = False
                    close_brackets = False
                else:
                    if close_brackets:
                        result = result.parent
                        close_brackets = False
                    call = Call(tok, c_type, self, result)
                    result.add_to_current_field(call)
                    result = call
            elif tok in brackets.keys():  # brackets
                level += 1
                # TODO why is this not working, when the two statements ar not
                # in the if/else clause (they are exactly the same!!!)
                if is_call_or_close():
                    result = Array(brackets[tok], self, result)
                    result = result.parent.add_execution(result)
                    close_brackets = False
                else:
                    result = Array(brackets[tok], self, result)
                    result.parent.add_to_current_field(result)
            elif tok == ':':
                if is_call_or_close():
                    result = result.parent
                    close_brackets = False
                result.add_dictionary_key()
            elif tok == '.':
                if close_brackets and result.parent != top:
                    # only get out of the array, if it is a array execution
                    result = result.parent
                    close_brackets = False
                is_chain = True
            elif tok == ',':
                if is_call_or_close():
                    result = result.parent
                    close_brackets = False
                result.add_field()
                # important - it cannot be empty anymore
                if result.type == Array.NOARRAY:
                    result.type = Array.TUPLE
            elif tok in [')', '}', ']']:
                while is_call_or_close():
                    result = result.parent
                    close_brackets = False
                if tok == '}' and not len(result):
                    # this is a really special case - empty brackets {} are
                    # always dictionaries and not sets.
                    result.type = Array.DICT
                level -= 1
                #result = result.parent
                close_brackets = True
            else:
                if is_call_or_close():
                    result = result.parent
                    close_brackets = False
                result.add_to_current_field(tok)

            #print 'tok_end', tok_temp, result, close_brackets

        if level != 0:
            raise ParserError("Brackets don't match: %s. This is not normal "
                                "behaviour. Please submit a bug" % level)

        self._assignment_calls = top
        return top


class Param(Statement):
    """
    The class which shows definitions of params of classes and functions.
    But this is not to define function calls.
    """
    def __init__(self, code, set_vars, used_funcs, used_vars, token_list,
            indent, line_nr, line_end):
        super(Param, self).__init__(code, set_vars, used_funcs,
                        used_vars, token_list, indent, line_nr, line_end)

        # this is defined by the parser later on, not at the initialization
        self.position = None

    def get_name(self):
        """ get the name of the param """
        n = self.set_vars or self.used_vars
        if len(n) > 1:
            raise IndexError("Multiple param names (%s)." % n)
        return n[0]


class Call(object):
    """
    TODO doc
    """
    NAME = object()
    NUMBER = object()
    STRING = object()

    """ The statement object of functions, to  """
    def __init__(self, name, type, parent_stmt, parent=None):
        self.name = name
        # parent is not the oposite of next. The parent of c: a = [b.c] would
        # be an array.
        self.parent = parent
        self.type = type

        self.next = None
        self.execution = None
        self.parent_stmt = parent_stmt

    def set_next_chain_call(self, call):
        """ Adds another part of the statement"""
        self.next = call
        #print '\n\npar', call.parent, self.parent, type(call), type(self)
        call.parent = self.parent
        return call

    def add_execution(self, call):
        """
        An execution is nothing else than brackets, with params in them, which
        shows access on the internals of this name.
        """
        self.execution = call
        # there might be multiple executions, like a()[0], in that case, they
        # have the same parent. Otherwise it's not possible to parse proper.
        if self.parent.execution == self:
            call.parent = self.parent
        else:
            call.parent = self
        return call

    def generate_call_list(self):
        try:
            for name_part in self.name.names:
                yield name_part
        except AttributeError:
            yield self
        if self.execution is not None:
            for y in self.execution.generate_call_list():
                yield y
        if self.next is not None:
            for y in self.next.generate_call_list():
                yield y

    def __repr__(self):
        return "<%s: %s of %s>" % \
                (self.__class__.__name__, self.name, self.parent)


class Array(Call):
    """
    Describes the different python types for an array, but also empty
    statements. In the Python syntax definitions this type is named 'atom'.
    http://docs.python.org/release/3.0.1/reference/grammar.html
    Array saves sub-arrays as well as normal operators and calls to methods.

    :param array_type: The type of an array, which can be one of the constants\
    below.
    :type array_type: int
    """
    NOARRAY = None
    TUPLE = 'tuple'
    LIST = 'list'
    DICT = 'dict'
    SET = 'set'

    def __init__(self, arr_type, parent_stmt, parent=None):
        super(Array, self).__init__(None, arr_type, parent_stmt, parent)

        self.values = []
        self.keys = []

    def add_field(self):
        """
        Just add a new field to the values.

        Each value has a sub-array, because there may be different tokens in
        one array.
        """
        self.values.append([])
        self.keys.append(None)

    def add_to_current_field(self, tok):
        """ Adds a token to the latest field (in content). """
        if not self.values:
            # add the first field, this is done here, because if nothing
            # gets added, the list is empty, which is also needed sometimes.
            self.values.append([])
        self.values[-1].append(tok)

    def add_dictionary_key(self):
        """
        Only used for dictionaries, automatically adds the tokens added by now
        from the values to keys.
        """
        self.type = Array.DICT
        c = self._counter
        self.keys[c] = self.values[c]
        self.values[c] = []

    def get_only_subelement(self):
        """
        Returns the only element that an array contains. If it contains
        more than one element, raise an exception.
        """
        if len(self.values) != 1 or len(self.values[0]) != 1:
            raise AttributeError("More than one value found")
        return self.values[0][0]

    @staticmethod
    def is_type(instance, typ):
        """
        This is not only used for calls on the actual object, but for
        ducktyping, to invoke this function with anything as `self`.
        """
        if isinstance(instance, Array):
            if instance.type == typ:
                return True
        return False

    def __len__(self):
        return len(self.values)

    def __getitem__(self, key):
        return self.values[key]

    def __iter__(self):
        if self.type == self.DICT:
            return self.values.items().__iter__()
        else:
            return self.values.__iter__()

    def __repr__(self):
        if self.type == self.NOARRAY:
            temp = 'noarray'
        else:
            temp = self.type
        parent_str = " of %s" % self.parent if self.parent else ""
        return "<%s: %s%s>" % (self.__class__.__name__, temp, parent_str)


class NamePart(str):
    """
    A string. Sometimes it is important to know if the string belongs to a name
    or not.
    """
    pass


class Name(Simple):
    """
    Used to define names in python.
    Which means the whole namespace/class/function stuff.
    So a name like "module.class.function"
    would result in an array of [module, class, function]
    """
    def __init__(self, names, indent, line_nr, line_end):
        super(Name, self).__init__(indent, line_nr, line_end)
        self.names = tuple(NamePart(n) for n in names)

    def get_code(self):
        """ Returns the names in a full string format """
        return ".".join(self.names)

    def __str__(self):
        return self.get_code()

    def __eq__(self, other):
        return self.names == other.names \
                and self.indent == other.indent \
                and self.line_nr == self.line_nr

    def __ne__(self, other):
        return not self.__eq__(other)

    def __hash__(self):
        return hash(self.names) + hash(self.indent) + hash(self.line_nr)

    def __len__(self):
        return len(self.names)


class PyFuzzyParser(object):
    """
    This class is used to parse a Python file, it then divides them into a
    class structure of different scopes.

    :param code: The codebase for the parser.
    :type code: str
    :param user_line: The line, the user is currently on.
    :type user_line: int
    """
    def __init__(self, code, module_path=None, user_line=None):
        self.user_line = user_line
        self.code = code + '\n'  # end with \n, because the parser needs it

        # initialize global Scope
        self.top = GlobalScope(module_path)
        self.scope = self.top
        self.current = (None, None, None)

        self._tokenize_line_nr = 0
        self._line_of_tokenize_restart = 0

        self.parse()

        # delete code again, only the parser needs it
        del self.code

    @property
    def line_nr(self):
        return self._line_of_tokenize_restart + self._tokenize_line_nr

    def _parsedotname(self, pre_used_token=None):
        """
        The dot name parser parses a name, variable or function and returns
        their names.

        :return: list of the names, token_type, nexttoken, start_indent, \
        start_line.
        :rtype: (Name, int, str, int, int)
        """
        names = []
        if pre_used_token is None:
            token_type, tok, indent = self.next()
            start_line = self.line_nr
            if token_type != tokenize.NAME and tok != '*':
                return ([], token_type, tok, indent, start_line)
        else:
            token_type, tok, indent = pre_used_token
            start_line = self.line_nr
        names.append(tok)
        start_indent = indent
        while True:
            token_type, tok, indent = self.next()
            if tok != '.':
                break
            token_type, tok, indent = self.next()
            if token_type != tokenize.NAME:
                break
            names.append(tok)
        return (names, token_type, tok, start_indent, start_line)

    def _parseimportlist(self):
        """
        The parser for the imports. Unlike the class and function parse
        function, this returns no Import class, but rather an import list,
        which is then added later on.
        The reason, why this is not done in the same class lies in the nature
        of imports. There are two ways to write them:

        - from ... import ...
        - import ...

        To distinguish, this has to be processed after the parser.

        :return: List of imports.
        :rtype: list
        """
        imports = []
        while True:
            name, token_type, tok, start_indent, start_line = \
                self._parsedotname()
            if not name:
                break
            name2 = None
            if tok == 'as':
                name2, token_type, tok, start_indent2, start_line = \
                    self._parsedotname()
                name2 = Name(name2, start_indent2, start_line, self.line_nr)
            i = Name(name, start_indent, start_line, self.line_nr)
            imports.append((i, name2))
            while tok not in [",", ";", "\n"]:
                token_type, tok, indent = self.next()
            if tok != ",":
                break
        return imports

    def _parseparen(self):
        """
        Functions and Classes have params (which means for classes
        super-classes). They are parsed here and returned as Statements.

        :return: List of Statements
        :rtype: list
        """
        names = []
        tok = None
        pos = 0
        while tok not in [')', ':']:
            stmt, tok = self._parse_statement(added_breaks=',',
                                              stmt_class=Param)
            if stmt:
                stmt.position = pos
                names.append(stmt)
                pos += 1

        return names

    def _parsefunction(self, indent):
        """
        The parser for a text functions. Process the tokens, which follow a
        function definition.

        :return: Return a Scope representation of the tokens.
        :rtype: Function
        """
        start_line = self.line_nr
        token_type, fname, ind = self.next()
        if token_type != tokenize.NAME:
            return None

        fname = Name([fname], ind, self.line_nr, self.line_nr)

        token_type, open, ind = self.next()
        if open != '(':
            return None
        params = self._parseparen()

        token_type, colon, ind = self.next()
        if colon != ':':
            return None

        return Function(fname, params, indent, start_line)

    def _parseclass(self, indent):
        """
        The parser for a text class. Process the tokens, which follow a
        class definition.

        :return: Return a Scope representation of the tokens.
        :rtype: Class
        """
        start_line = self.line_nr
        token_type, cname, ind = self.next()
        if token_type != tokenize.NAME:
            debug.dbg("class: syntax error - token is not a name@%s (%s: %s)" \
                    % (self.line_nr, tokenize.tok_name[token_type], cname))
            return None

        cname = Name([cname], ind, self.line_nr, self.line_nr)

        super = []
        token_type, next, ind = self.next()
        if next == '(':
            super = self._parseparen()
            token_type, next, ind = self.next()

        if next != ':':
            debug.dbg("class: syntax error - %s@%s" % (cname, self.line_nr))
            return None

        return Class(cname, super, indent, start_line)

    def _parse_statement(self, pre_used_token=None, added_breaks=None,
                            stmt_class=Statement):
        """
        Parses statements like:

        >>> a = test(b)
        >>> a += 3 - 2 or b

        and so on. One row at a time.

        :param pre_used_token: The pre parsed token.
        :type pre_used_token: set
        :return: Statement + last parsed token.
        :rtype: (Statement, str)

        TODO improve abort criterion of not closing parentheses
        """

        string = ''
        set_vars = []
        used_funcs = []
        used_vars = []
        level = 0  # The level of parentheses
        is_return = None

        if pre_used_token:
            token_type, tok, indent = pre_used_token
        else:
            token_type, tok, indent = self.next()

        line_start = self.line_nr

        # the difference between "break" and "always break" is that the latter
        # will even break in parentheses. This is true for typical flow
        # commands like def and class and the imports, which will never be used
        # in a statement.
        breaks = ['\n', ':', ')']
        always_break = [';', 'import', 'from', 'class', 'def', 'try', 'except',
                        'finally', 'while']
        if added_breaks:
            breaks += added_breaks

        tok_list = []
        while not (tok in always_break or tok in breaks and level <= 0):
            set_string = None
            #print 'parse_stmt', tok, tokenize.tok_name[token_type]
            tok_list.append(self.current)
            if tok == 'as':
                string += " %s " % tok
                token_type, tok, indent_dummy = self.next()
                if token_type == tokenize.NAME:
                    path, token_type, tok, start_indent, start_line = \
                            self._parsedotname(self.current)
                    n = Name(path, start_indent, start_line, self.line_nr)
                    set_vars.append(n)
                    tok_list.append(n)
                    string += ".".join(path)
                continue
            elif token_type == tokenize.NAME:
                #print 'is_name', tok
                if tok in ['return', 'yield', 'del', 'raise', 'assert']:
                    if len(tok_list) > 1:
                        # this happens, when a statement has opening brackets,
                        # which are not closed again, here I just start a new
                        # statement. This is a hack, but I could not come up
                        # with a better solution.
                        # This is basically a reset of the statement.
                        debug.warning('keyword in statement @%s', tok_list,
                                        self.line_nr)
                        tok_list = [self.current]
                        set_vars = []
                        used_funcs = []
                        used_vars = []
                        level = 0
                    set_string = tok + ' '
                    if tok in ['return', 'yield']:
                        is_return = tok
                elif tok in ['print', 'exec']:
                    # delete those statements, just let the rest stand there
                    set_string = ''
                else:
                    path, token_type, tok, start_indent, start_line = \
                            self._parsedotname(self.current)
                    n = Name(path, start_indent, start_line, self.line_nr)
                    tok_list.pop()  # removed last entry, because we add Name
                    tok_list.append(n)
                    if tok == '(':
                        # it must be a function
                        used_funcs.append(n)
                    else:
                        used_vars.append(n)
                    if string and re.match(r'[\w\d\'"]', string[-1]):
                        string += ' '
                    string += ".".join(path)
                    #print 'parse_stmt', tok, tokenize.tok_name[token_type]
                    continue
            elif '=' in tok and not tok in ['>=', '<=', '==', '!=']:
                # there has been an assignement -> change vars
                if level == 0:
                    set_vars = used_vars
                    used_vars = []
            elif tok in ['{', '(', '[']:
                level += 1
            elif tok in ['}', ')', ']']:
                level -= 1

            if set_string is not None:
                string = set_string
            else:
                string += tok
            # caution: don't use indent anywhere,
            # it's not working with the name parsing
            token_type, tok, indent_dummy = self.next()
        if not string:
            return None, tok
        #print 'new_stat', string, set_vars, used_funcs, used_vars
        if self.freshscope and len(tok_list) > 1 \
                    and self.last_token[1] == tokenize.STRING:
            self.scope.add_docstr(self.last_token[1])
        else:
            stmt = stmt_class(string, set_vars, used_funcs, used_vars,\
                                tok_list, indent, line_start, self.line_nr)
        if is_return:
            # add returns to the scope
            func = self.scope.get_parent_until(Function)
            if is_return == 'yield':
                func.is_generator = True
            try:
                func.returns.append(stmt)
            except AttributeError:
                debug.warning('return in non-function')

        return stmt, tok

    def next(self):
        """ Generate the next tokenize pattern. """
        type, tok, position, dummy, self.parserline = self.gen.next()
        (self._tokenize_line_nr, indent) = position
        if self.line_nr == self.user_line:
            debug.dbg('user scope found [%s] =%s' % \
                    (self.parserline.replace('\n', ''), repr(self.scope)))
            self.user_scope = self.scope
        self.last_token = self.current
        self.current = (type, tok, indent)
        return self.current

    def parse(self):
        """
        The main part of the program. It analyzes the given code-text and
        returns a tree-like scope. For a more detailed description, see the
        class description.

        :param text: The code which should be parsed.
        :param type: str

        :raises: IndentationError
        """
        buf = cStringIO.StringIO(self.code)
        self.gen = tokenize.generate_tokens(buf.readline)
        self.currentscope = self.scope

        extended_flow = ['else', 'elif', 'except', 'finally']
        statement_toks = ['{', '[', '(', '`']

        decorators = []
        self.freshscope = True
        while True:
            try:
                token_type, tok, indent = self.next()
                #debug.dbg('main: tok=[%s] type=[%s] indent=[%s]'\
                #    % (tok, token_type, indent))

                while token_type == tokenize.DEDENT and self.scope != self.top:
                    debug.dbg('dedent', self.scope)
                    token_type, tok, indent = self.next()
                    if indent <= self.scope.indent:
                        self.scope.line_end = self.line_nr
                        self.scope = self.scope.parent

                # check again for unindented stuff. this is true for syntax
                # errors. only check for names, because thats relevant here. If
                # some docstrings are not indented, I don't care.
                while indent <= self.scope.indent \
                        and (token_type == tokenize.NAME or tok in ['(', '['])\
                        and self.scope != self.top:
                    debug.dbg('syntax: dedent @%s - %s<=%s', \
                            (self.line_nr, indent, self.scope.indent))
                    self.scope.line_end = self.line_nr
                    self.scope = self.scope.parent

                start_line = self.line_nr
                if tok == 'def':
                    func = self._parsefunction(indent)
                    if func is None:
                        debug.warning("function: syntax error@%s" %
                                            self.line_nr)
                        continue
                    debug.dbg("new scope: function %s" % (func.name))
                    self.freshscope = True
                    self.scope = self.scope.add_scope(func, decorators)
                    decorators = []
                elif tok == 'class':
                    cls = self._parseclass(indent)
                    if cls is None:
                        debug.warning("class: syntax error@%s" %
                                            self.line_nr)
                        continue
                    self.freshscope = True
                    debug.dbg("new scope: class %s" % (cls.name))
                    self.scope = self.scope.add_scope(cls, decorators)
                    decorators = []
                # import stuff
                elif tok == 'import':
                    imports = self._parseimportlist()
                    for m, alias in imports:
                        i = Import(indent, start_line, self.line_nr, m, alias)
                        self.scope.add_import(i)
                        debug.dbg("new import: %s" % (i), self.current)
                    self.freshscope = False
                elif tok == 'from':
                    # take care for relative imports
                    relative_count = 0
                    while 1:
                        token_type, tok, indent = self.next()
                        if tok != '.':
                            break
                        relative_count += 1
                    # the from import
                    mod, token_type, tok, start_indent, start_line2 = \
                        self._parsedotname(self.current)
                    if not mod or tok != "import":
                        debug.warning("from: syntax error@%s" %
                                            self.line_nr)
                        continue
                    mod = Name(mod, start_indent, start_line2, self.line_nr)
                    names = self._parseimportlist()
                    for name, alias in names:
                        star = name.names[0] == '*'
                        if star:
                            name = None
                        i = Import(indent, start_line, self.line_nr, name,
                                    alias, mod, star, relative_count)
                        self.scope.add_import(i)
                        debug.dbg("new from: %s" % (i))
                    self.freshscope = False
                #loops
                elif tok == 'for':
                    set_stmt, tok = self._parse_statement(added_breaks=['in'])
                    if tok == 'in':
                        statement, tok = self._parse_statement()
                        if tok == ':':
                            f = ForFlow('for', [statement], indent,
                                            self.line_nr, set_stmt)
                            debug.dbg("new scope: flow for@%s" % (f.line_nr))
                            self.scope = self.scope.add_statement(f)

                elif tok in ['if', 'while', 'try', 'with'] + extended_flow:
                    added_breaks = []
                    command = tok
                    if command in ['except', 'with']:
                        added_breaks.append(',')
                    # multiple statements because of with
                    inits = []
                    first = True
                    while first or command == 'with' and tok != ':':
                        statement, tok = \
                            self._parse_statement(added_breaks=added_breaks)
                        if command == 'except' and tok in added_breaks:
                            # the except statement defines a var
                            # this is only true for python 2
                            path, token_type, tok, start_indent, start_line2 =\
                                    self._parsedotname()
                            n = Name(path, start_indent, start_line2,
                                                            self.line_nr)
                            statement.set_vars.append(n)
                            statement.code += ',' + n.get_code()
                        if statement:
                            inits.append(statement)
                        first = False

                    if tok == ':':
                        f = Flow(command, inits, indent, self.line_nr)
                        debug.dbg("new scope: flow %s@%s"
                                        % (command, self.line_nr))
                        if command in extended_flow:
                            # the last statement has to be another part of
                            # the flow statement, because a dedent releases the
                            # main scope, so just take the last statement.
                            self.scope = self.scope.statements[-1].set_next(f)
                        else:
                            self.scope = self.scope.add_statement(f)
                    else:
                        debug.warning('syntax err, flow started @%s',
                                                            self.line_nr)
                # globals
                elif tok == 'global':
                    stmt, tok = self._parse_statement(self.current)
                    if stmt:
                        self.scope.add_statement(stmt)
                        debug.dbg('global_vars', stmt.used_vars)
                        for name in stmt.used_vars:
                            # add the global to the top, because there it is
                            # important.
                            self.top.add_global(name)
                # decorator
                elif tok == '@':
                    stmt, tok = self._parse_statement()
                    decorators.append(stmt)
                elif tok == 'pass':
                    continue
                # default
                elif token_type in [tokenize.NAME, tokenize.STRING,
                                    tokenize.NUMBER] \
                        or tok in statement_toks:
                    # this is the main part - a name can be a function or a
                    # normal var, which can follow anything. but this is done
                    # by the statement parser.
                    stmt, tok = self._parse_statement(self.current)
                    if stmt:
                        debug.dbg('new stmt', stmt)
                        self.scope.add_statement(stmt)
                    self.freshscope = False
                else:
                    if token_type not in [tokenize.COMMENT, tokenize.INDENT,
                                          tokenize.NEWLINE, tokenize.NL,
                                          tokenize.ENDMARKER]:
                        debug.warning('token not classified', tok, token_type,
                                        self.line_nr)
            except StopIteration:  # thrown on EOF
                break
            except tokenize.TokenError:
                # We just ignore this error, I try to handle it earlier - as
                # good as possible
                debug.warning('parentheses not closed error')
            except IndentationError:
                # This is an error, that tokenize may produce, because the code
                # is not indented as it should. Here it just ignores this line
                # and restarts the parser.
                # (This is a rather unlikely error message, for normal code,
                # tokenize seems to be pretty tolerant)
                self._line_of_tokenize_restart = self.line_nr + 1
                self._tokenize_line_nr = 0
                debug.warning('indentation error on line %s, ignoring it' %
                                (self.line_nr))
                self.gen = tokenize.generate_tokens(buf.readline)
        return self.top