diff --git a/evaluate.py b/evaluate.py index 6ee4763c..94a5bbbf 100644 --- a/evaluate.py +++ b/evaluate.py @@ -1,9 +1,5 @@ import parsing -def Statement(object): - """ The statement object of functions, to """ - pass - def get_names_for_scope(scope): """ Get all completions possible for the current scope. """ comp = [] @@ -38,6 +34,7 @@ def follow_path(scope, path): scopes.append(c.parent) #elif p_class == parsing.Function: elif p_class == parsing.Statement: + print 'state', c.parent.token_list, c.parent.get_assignment_calls() pass else: print 'error follow_path:', p_class, repr(c.parent) @@ -49,6 +46,42 @@ def follow_path(scope, path): scopes = new_scopes return set(scopes) +def _parseassignment(self): + """ TODO remove or replace, at the moment not used """ + assign = '' + token_type, tok, indent = self.next() + if token_type == tokenize.STRING or tok == 'str': + return '""' + elif tok == '(' or tok == 'tuple': + return '()' + elif tok == '[' or tok == 'list': + return '[]' + elif tok == '{' or tok == 'dict': + return '{}' + elif token_type == tokenize.NUMBER: + return '0' + elif tok == 'open' or tok == 'file': + return 'file' + elif tok == 'None': + return '_PyCmplNoType()' + elif tok == 'type': + return 'type(_PyCmplNoType)' # only for method resolution + else: + assign += tok + level = 0 + while True: + token_type, tok, indent = self.next() + if tok in ('(', '{', '['): + level += 1 + elif tok in (']', '}', ')'): + level -= 1 + if level == 0: + break + elif level == 0: + if tok in (';', '\n'): + break + assign += tok + return "%s" % assign def dbg(*args): if debug_function: diff --git a/functions.py b/functions.py index 6572e6d4..a623928f 100644 --- a/functions.py +++ b/functions.py @@ -222,11 +222,15 @@ def dbg(*args): debug_function(*args) -def set_debug_function(func): +def set_debug_function(func_cb): + """ + You can define a callback debug function to get all the debug messages. + :param func_cb: The callback function for debug messages, with n params. + """ global debug_function - debug_function = func - parsing.debug_function = func - evaluate.debug_function = func + debug_function = func_cb + parsing.debug_function = func_cb + evaluate.debug_function = func_cb debug_function = None diff --git a/parsing.py b/parsing.py index 6890a8c5..9531683d 100644 --- a/parsing.py +++ b/parsing.py @@ -39,6 +39,14 @@ import cStringIO import re +class TokenNotFoundError(Exception): + pass + + +class ParserError(Exception): + pass + + def indent_block(text, indention=" "): """ This function indents a text block with a default of four spaces """ temp = '' @@ -460,18 +468,21 @@ class Statement(Simple): :param used_funcs: str :param used_vars: The variables which are used by the statement. :param used_vars: str + :param token_list: Token list which is also peppered with Name. + :param token_list: list :param indent: The indent level of the flow statement. :type indent: int :param line_nr: Line number of the flow statement. :type line_nr: int """ - def __init__(self, code, set_vars, used_funcs, used_vars, indent, line_nr, - line_end): + def __init__(self, code, set_vars, used_funcs, used_vars, token_list, + indent, line_nr, line_end): super(Statement, self).__init__(indent, line_nr, line_end) self.code = code self.set_vars = set_vars self.used_funcs = used_funcs self.used_vars = used_vars + self.token_list = token_list for s in set_vars + used_funcs + used_vars: s.parent = self @@ -485,6 +496,170 @@ class Statement(Simple): """ Get the names for the statement. """ return self.set_vars + def get_assignment_calls(self): + """ + This is not done in the main parser, because it might be slow and + most of the statements won't need this data anyway. This is something + 'like' a lazy execution. + """ + result = None + has_assignment = False + level = 0 + is_chain = False + + for tok_temp in self.token_list: + print 'tok', tok_temp + try: + token_type, tok, indent = tok_temp + if '=' in tok and not tok in ['>=', '<=', '==', '!=']: + # This means, there is an assignment here. + # TODO there may be multiple assignments: a = b = 1 + has_assignment = True + + # initialize the first item + result = Array(Array.EMPTY) + continue + except TypeError: + # the token is a Name, which has already been parsed + tok = tok_temp + + if has_assignment: + brackets = {'(': Array.EMPTY, '[': Array.LIST, '{': Array.SET} + is_call = isinstance(result, Call) + if isinstance(tok, Name): + call = Call(tok, result) + if is_chain: + result = result.set_next_chain_call(call) + is_chain = False + else: + result.add_to_current_field(call) + result = call + print 'asdf', result, result.parent + elif tok in brackets.keys(): + level += 1 + result = Array(brackets[tok], result) + if is_call: + result = result.parent.add_execution(result) + else: + result.parent.add_to_current_field(result) + elif tok == ':': + if is_call: + result = result.parent + result.add_dictionary_key() + elif tok == '.': + is_chain = True + elif tok == ',': + if is_call: + result = result.parent + result.add_field() + # important - it cannot be empty anymore + if result.arr_type == Array.EMPTY: + result.arr_type = Array.TUPLE + elif tok in [')', '}', ']']: + level -= 1 + print 'asdf2', result, result.parent + result = result.parent + else: + # TODO catch numbers and strings -> token_type and make + # calls out of them + if is_call: + result = result.parent + result.add_to_current_field(tok) + + if not has_assignment: + raise TokenNotFoundError("You are requesting the result of an " + "assignment, where the token cannot be found") + if level != 0: + raise ParserError("Brackets don't match: %s. This is not normal " + "behaviour. Please submit a bug" % level) + return result + + +class Array(object): + """ + Describes the different python types for an array, but also empty + statements. In the Python syntax definitions this type is named 'atom'. + http://docs.python.org/release/3.0.1/reference/grammar.html + Array saves sub-arrays as well as normal operators and calls to methods. + + :param array_type: The type of an array, which can be one of the constants\ + below. + :type array_type: int + """ + EMPTY = object() + TUPLE = object() + LIST = object() + DICT = object() + SET = object() + + def __init__(self, arr_type, parent=None): + self.arr_type = arr_type + self.values = [] + self.keys = [] + self.parent = parent + + def add_field(self): + """ + Just add a new field to the values. + """ + self.values.append([]) + self.keys.append(None) + + def add_to_current_field(self, tok): + """ Adds a token to the latest field (in content). """ + if not self.values: + # add the first field, this is done here, because if nothing + # gets added, the list is empty, which is also needed sometimes. + self.values.append([]) + self.values[-1].append(tok) + + def add_dictionary_key(self): + """ + Only used for dictionaries, automatically adds the tokens added by now + from the values to keys. + """ + self.arr_type = Array.DICT + c = self._counter + self.keys[c] = self.values[c] + self.values[c] = [] + + def __len__(self): + return len(self.values) + + def __getitem__(self, key): + return self.values[key] + + def __iter__(self): + if self.arr_type == self.DICT: + return self.values.items() + else: + return self.values + + +class Call(object): + """ The statement object of functions, to """ + def __init__(self, name, parent): + self.name = name + self.parent = parent + + self.next = None + self.param_array = None + self.executions = [] + + def set_next_chain_call(self, call): + """ Adds another part of the statement""" + self.next = call + call.parent = self.parent + return call + + def add_execution(self, call): + """ + An execution is nothing else than brackets, with params in them, which + shows access on the internals of this name. + """ + self.executions.append(call) + return call + class Name(Simple): """ @@ -704,43 +879,6 @@ class PyFuzzyParser(object): return Class(cname, super, indent, start_line) - def _parseassignment(self): - """ TODO remove or replace, at the moment not used """ - assign = '' - token_type, tok, indent = self.next() - if token_type == tokenize.STRING or tok == 'str': - return '""' - elif tok == '(' or tok == 'tuple': - return '()' - elif tok == '[' or tok == 'list': - return '[]' - elif tok == '{' or tok == 'dict': - return '{}' - elif token_type == tokenize.NUMBER: - return '0' - elif tok == 'open' or tok == 'file': - return 'file' - elif tok == 'None': - return '_PyCmplNoType()' - elif tok == 'type': - return 'type(_PyCmplNoType)' # only for method resolution - else: - assign += tok - level = 0 - while True: - token_type, tok, indent = self.next() - if tok in ('(', '{', '['): - level += 1 - elif tok in (']', '}', ')'): - level -= 1 - if level == 0: - break - elif level == 0: - if tok in (';', '\n'): - break - assign += tok - return "%s" % assign - def _parse_statement(self, pre_used_token=None, added_breaks=None): """ Parses statements like: @@ -778,9 +916,11 @@ class PyFuzzyParser(object): if added_breaks: breaks += added_breaks + tok_list = [] while not (tok in always_break or tok in breaks and level <= 0): set_string = None #print 'parse_stmt', tok, tokenize.tok_name[token_type] + tok_list.append(self.current) if tok == 'as': string += " %s " % tok token_type, tok, indent_dummy = self.next() @@ -789,6 +929,7 @@ class PyFuzzyParser(object): self._parsedotname(self.current) n = Name(path, start_indent, start_line, self.line_nr) set_vars.append(n) + tok_list.append(n) string += ".".join(path) continue elif token_type == tokenize.NAME: @@ -802,12 +943,13 @@ class PyFuzzyParser(object): path, token_type, tok, start_indent, start_line = \ self._parsedotname(self.current) n = Name(path, start_indent, start_line, self.line_nr) + tok_list.pop() # remove last entry, because we add Name + tok_list.append(n) if tok == '(': # it must be a function used_funcs.append(n) else: - if not n.names[0] in ['global']: - used_vars.append(n) + used_vars.append(n) if string and re.match(r'[\w\d\'"]', string[-1]): string += ' ' string += ".".join(path) @@ -833,7 +975,7 @@ class PyFuzzyParser(object): return None, tok #print 'new_stat', string, set_vars, used_funcs, used_vars stmt = Statement(string, set_vars, used_funcs, used_vars,\ - indent, line_start, self.line_nr) + tok_list, indent, line_start, self.line_nr) return stmt, tok def next(self): @@ -887,7 +1029,7 @@ class PyFuzzyParser(object): while indent <= self.scope.indent \ and token_type in [tokenize.NAME] \ and self.scope != self.top: - dbg( 'syntax_err, dedent @%s - %s<=%s', \ + dbg('syntax_err, dedent @%s - %s<=%s', \ (self.line_nr, indent, self.scope.indent)) self.scope.line_end = self.line_nr self.scope = self.scope.parent diff --git a/test.py b/test.py index 83158166..9d6d5b82 100644 --- a/test.py +++ b/test.py @@ -128,11 +128,11 @@ if True or a: # completion import time class c1(): - c2 = c1() + c2,c5 = c1(), c1().c3() def c3(self): import time as c4 c5 = 3 - + return 'asdf'