diff --git a/builtin.py b/builtin.py index c329f622..fd84934e 100644 --- a/builtin.py +++ b/builtin.py @@ -124,10 +124,13 @@ class Parser(object): value = '%s.%s' % (mod, value) code += '%s = %s\n' % (name, value) - if depth == 10: - import sys - sys.stdout.write(code) - exit() + if depth == 0: + #with open('writeout.py', 'w') as f: + # f.write(code) + #import sys + #sys.stdout.write(code) + #exit() + pass return code diff --git a/evaluate.py b/evaluate.py index 42d2b0c1..308fb490 100644 --- a/evaluate.py +++ b/evaluate.py @@ -45,6 +45,9 @@ class Instance(Exec): n += self.base.get_set_vars() return n + def get_defined_names(self): + return self.get_set_vars() + def __repr__(self): return "<%s of %s>" % \ (self.__class__.__name__, self.base) @@ -107,6 +110,7 @@ def get_names_for_scope(scope): # add builtins to the global scope compl += builtin.Builtin.scope.get_set_vars() + #print 'gnfs', scope, compl return compl @@ -136,15 +140,15 @@ def get_scopes_for_name(scope, name, search_global=False): # the name is already given in the parent function result = [] for scope in scopes: - if isinstance(scope, parsing.Import): - try: - debug.dbg('star import', scope) - i = follow_import(scope).get_defined_names() - except modules.ModuleNotFound: - debug.dbg('StarImport not found: ' + str(scope)) - else: - result += filter_name(i) - else: + #if isinstance(scope, parsing.Import): + # try: + # debug.dbg('star import', scope) + # i = follow_import(scope).get_defined_names() + # except modules.ModuleNotFound: + # debug.dbg('StarImport not found: ' + str(scope)) + # else: + # result += filter_name(i) + #else: if [name] == list(scope.names): result.append(scope.parent) debug.dbg('sfn filter', result) @@ -158,24 +162,22 @@ def get_scopes_for_name(scope, name, search_global=False): return remove_statements(filter_name(names)) -def resolve_results(scopes): - """ Here we follow the results - to get what we really want """ +def strip_imports(scopes): + """ + Here we strip the imports - they don't get resolved necessarily, but star + imports are looked at here. + """ result = [] for s in scopes: if isinstance(s, parsing.Import): print 'dini mueter, steile griech!' try: scope = follow_import(s) - #for r in resolve_results([follow_import(s)]): - # if isinstance(r, parsing.Import): - # resolve_results(r) - # else: - # resolve except modules.ModuleNotFound: debug.dbg('Module not found: ' + str(s)) else: result.append(scope) - result += resolve_results(i for i in scope.get_imports() if i.star) + result += strip_imports(i for i in scope.get_imports() if i.star) else: result.append(s) return result @@ -196,8 +198,6 @@ def follow_statement(stmt, scope=None): if not isinstance(tok, str): # the string tokens are just operations (+, -, etc.) result += follow_call(scope, tok) - else: - debug.warning('dini mueter, found string:', tok) return result @@ -209,8 +209,15 @@ def follow_call(scope, call): if isinstance(current, parsing.Array): result = [current] else: - scopes = get_scopes_for_name(scope, current, search_global=True) - result = resolve_results(scopes) + # TODO add better care for int/unicode, now str/float are just used + # instead + if current.type == parsing.Call.STRING: + scopes = get_scopes_for_name(builtin.Builtin.scope, 'str') + elif current.type == parsing.Call.NUMBER: + scopes = get_scopes_for_name(builtin.Builtin.scope, 'float') + else: + scopes = get_scopes_for_name(scope, current, search_global=True) + result = strip_imports(scopes) debug.dbg('call before', result, current, scope) result = follow_paths(path, result) @@ -245,12 +252,12 @@ def follow_path(path, input): result = [] if isinstance(current, parsing.Array): # this must be an execution, either () or [] - if current.arr_type == parsing.Array.LIST: + if current.type == parsing.Array.LIST: result = [] # TODO eval lists - elif current.arr_type not in [parsing.Array.DICT, parsing]: + elif current.type not in [parsing.Array.DICT, parsing]: # scope must be a class or func - make an instance or execution debug.dbg('befexec', scope) - result = resolve_results(Execution(scope).get_return_types()) + result = strip_imports(Execution(scope).get_return_types()) debug.dbg('exec', result) #except AttributeError: # debug.dbg('cannot execute:', scope) @@ -263,7 +270,7 @@ def follow_path(path, input): result = [] else: # TODO check magic class methods and return them also - result = resolve_results(get_scopes_for_name(scope, current)) + result = strip_imports(get_scopes_for_name(scope, current)) return result return follow_paths(path, filter_result(input)) diff --git a/functions.py b/functions.py index 5866a79b..31bbbf03 100644 --- a/functions.py +++ b/functions.py @@ -75,13 +75,18 @@ class FileWithCursor(modules.File): force_point = False elif force_point: if tok != '.': - break + # it is reversed, therefore a number is getting recognized + # as a floating point number + if not (token_type == tokenize.NUMBER and tok[0] == '.'): + #print 'break2', token_type, tok + break elif tok in close_brackets: level += 1 elif token_type in [tokenize.NAME, tokenize.STRING, tokenize.NUMBER]: force_point = True else: + #print 'break', token_type, tok break string += tok @@ -115,7 +120,7 @@ def complete(source, row, column, file_callback=None): column = 17 row = 140 - row = 148 + row = 150 column = 200 f = FileWithCursor('__main__', source=source, row=row) scope = f.parser.user_scope @@ -129,6 +134,7 @@ def complete(source, row, column, file_callback=None): try: path = f.get_row_path(column) + print path debug.dbg('completion_path', path) except ParserError as e: path = [] @@ -137,7 +143,9 @@ def complete(source, row, column, file_callback=None): result = [] if path and path[0]: # just parse one statement + #debug.ignored_modules = ['builtin'] r = parsing.PyFuzzyParser(path) + #debug.ignored_modules = ['parsing', 'builtin'] #print 'p', r.top.get_code().replace('\n', r'\n'), r.top.statements[0] scopes = evaluate.follow_statement(r.top.statements[0], scope) diff --git a/parsing.py b/parsing.py index ed6ebca4..39de7179 100644 --- a/parsing.py +++ b/parsing.py @@ -515,6 +515,7 @@ class Statement(Simple): close_brackets = False debug.dbg('tok_list', self.token_list) + print 'tok_list', self.token_list for i, tok_temp in enumerate(self.token_list): #print 'tok', tok_temp, result try: @@ -531,13 +532,21 @@ class Statement(Simple): except TypeError: # the token is a Name, which has already been parsed tok = tok_temp + token_type = None brackets = {'(': Array.EMPTY, '[': Array.LIST, '{': Array.SET} is_call = lambda: result.__class__ == Call is_call_or_close = lambda: is_call() or close_brackets - if isinstance(tok, Name): # names + if isinstance(tok, Name) or token_type in [tokenize.STRING, + tokenize.NUMBER]: # names + c_type = Call.NAME + if token_type == tokenize.STRING: + c_type = Call.STRING + elif token_type == tokenize.NUMBER: + c_type = Call.NUMBER + if is_chain: - call = Call(tok, result) + call = Call(tok, c_type, result) result = result.set_next_chain_call(call) is_chain = False close_brackets = False @@ -545,7 +554,7 @@ class Statement(Simple): if close_brackets: result = result.parent close_brackets = False - call = Call(tok, result) + call = Call(tok, c_type, result) result.add_to_current_field(call) result = call elif tok in brackets.keys(): # brackets @@ -573,8 +582,8 @@ class Statement(Simple): close_brackets = False result.add_field() # important - it cannot be empty anymore - if result.arr_type == Array.EMPTY: - result.arr_type = Array.TUPLE + if result.type == Array.EMPTY: + result.type = Array.TUPLE elif tok in [')', '}', ']']: while is_call_or_close(): result = result.parent @@ -582,22 +591,10 @@ class Statement(Simple): if tok == '}' and not len(result): # this is a really special case - empty brackets {} are # always dictionaries and not sets. - result.arr_type = Array.DICT + result.type = Array.DICT level -= 1 #result = result.parent close_brackets = True - elif tok in [tokenize.STRING, tokenize.NUMBER]: - # TODO catch numbers and strings -> token_type and make - # calls out of them - if is_call_or_close(): - result = result.parent - close_brackets = False - - call = Call(tok, result) - result.add_to_current_field(call) - result = call - result.add_to_current_field(tok) - pass else: if is_call_or_close(): result = result.parent @@ -611,16 +608,22 @@ class Statement(Simple): "behaviour. Please submit a bug" % level) self.assignment_calls = top + print 'top', top.values return top class Call(object): + NAME = object() + NUMBER = object() + STRING = object() + """ The statement object of functions, to """ - def __init__(self, name, parent=None): + def __init__(self, name, type, parent=None): self.name = name # parent is not the oposite of next. The parent of c: a = [b.c] would # be an array. self.parent = parent + self.type = type self.next = None self.execution = None @@ -682,9 +685,8 @@ class Array(Call): SET = object() def __init__(self, arr_type, parent=None): - super(Array, self).__init__(None, parent) + super(Array, self).__init__(None, arr_type, parent) - self.arr_type = arr_type self.values = [] self.keys = [] @@ -711,7 +713,7 @@ class Array(Call): Only used for dictionaries, automatically adds the tokens added by now from the values to keys. """ - self.arr_type = Array.DICT + self.type = Array.DICT c = self._counter self.keys[c] = self.values[c] self.values[c] = [] @@ -723,21 +725,21 @@ class Array(Call): return self.values[key] def __iter__(self): - if self.arr_type == self.DICT: + if self.type == self.DICT: return self.values.items().__iter__() else: return self.values.__iter__() def __repr__(self): - if self.arr_type == self.EMPTY: + if self.type == self.EMPTY: temp = 'empty' - elif self.arr_type == self.TUPLE: + elif self.type == self.TUPLE: temp = 'tuple' - elif self.arr_type == self.LIST: + elif self.type == self.LIST: temp = 'list' - elif self.arr_type == self.DICT: + elif self.type == self.DICT: temp = 'dict' - elif self.arr_type == self.SET: + elif self.type == self.SET: temp = 'set' return "<%s: %s of %s>" % \ (self.__class__.__name__, temp, self.parent) @@ -962,7 +964,9 @@ class PyFuzzyParser(object): token_type, next, ind = self.next() if next == '(': super = self._parseparen() - elif next != ':': + token_type, next, ind = self.next() + + if next != ':': debug.dbg("class: syntax error - %s@%s" % (cname, self.line_nr)) return None @@ -982,6 +986,7 @@ class PyFuzzyParser(object): :return: Statement + last parsed token. :rtype: (Statement, str) """ + string = '' set_vars = [] used_funcs = [] @@ -1035,7 +1040,7 @@ class PyFuzzyParser(object): path, token_type, tok, start_indent, start_line = \ self._parsedotname(self.current) n = Name(path, start_indent, start_line, self.line_nr) - tok_list.pop() # remove last entry, because we add Name + tok_list.pop() # removed last entry, because we add Name tok_list.append(n) if tok == '(': # it must be a function @@ -1066,8 +1071,12 @@ class PyFuzzyParser(object): if not string: return None, tok #print 'new_stat', string, set_vars, used_funcs, used_vars - stmt = Statement(string, set_vars, used_funcs, used_vars,\ - tok_list, indent, line_start, self.line_nr) + if self.freshscope and len(tok_list) > 1 \ + and self.last_token[1] == tokenize.STRING: + self.scope.add_docstr(self.last_token[1]) + else: + stmt = Statement(string, set_vars, used_funcs, used_vars,\ + tok_list, indent, line_start, self.line_nr) if is_return: # add returns to the scope func = self.scope.get_parent_until(Function) @@ -1108,7 +1117,7 @@ class PyFuzzyParser(object): statement_toks = ['{', '[', '(', '`'] decorators = [] - freshscope = True + self.freshscope = True while True: try: token_type, tok, indent = self.next() @@ -1141,7 +1150,7 @@ class PyFuzzyParser(object): self.line_nr) continue debug.dbg("new scope: function %s" % (func.name)) - freshscope = True + self.freshscope = True self.scope = self.scope.add_scope(func, decorators) decorators = [] elif tok == 'class': @@ -1150,7 +1159,7 @@ class PyFuzzyParser(object): debug.warning("class: syntax error@%s" % self.line_nr) continue - freshscope = True + self.freshscope = True debug.dbg("new scope: class %s" % (cls.name)) self.scope = self.scope.add_scope(cls, decorators) decorators = [] @@ -1160,7 +1169,7 @@ class PyFuzzyParser(object): for m, alias in imports: i = Import(indent, start_line, self.line_nr, m, alias) self.scope.add_import(i) - freshscope = False + self.freshscope = False elif tok == 'from': mod, token_type, tok, start_indent, start_line2 = \ self._parsedotname() @@ -1177,7 +1186,7 @@ class PyFuzzyParser(object): i = Import(indent, start_line, self.line_nr, name, alias, mod, star) self.scope.add_import(i) - freshscope = False + self.freshscope = False #loops elif tok == 'for': value_list, tok = self._parse_value_list() @@ -1230,23 +1239,26 @@ class PyFuzzyParser(object): decorators.append(stmt) elif tok == 'pass': continue - # check for docstrings - elif token_type == tokenize.STRING: - if freshscope: - self.scope.add_docstr(tok) - # this is the main part - a name can be a function or a normal - # var, which can follow anything. but this is done by the - # statement parser. - elif token_type == tokenize.NAME or tok in statement_toks: + # default + elif token_type in [tokenize.NAME, tokenize.STRING, + tokenize.NUMBER] \ + or tok in statement_toks: + # this is the main part - a name can be a function or a + # normal var, which can follow anything. but this is done + # by the statement parser. stmt, tok = self._parse_statement(self.current) if stmt: self.scope.add_statement(stmt) - freshscope = False - #else: - #print "_not_implemented_", tok, self.parserline + self.freshscope = False + else: + if token_type not in [tokenize.COMMENT, tokenize.INDENT, + tokenize.NEWLINE, tokenize.NL, + tokenize.ENDMARKER]: + debug.warning('token not classified', tok, token_type, + self.line_nr) except StopIteration: # thrown on EOF break - #except StopIteration: + #except: # debug.dbg("parse error: %s, %s @ %s" % # (sys.exc_info()[0], sys.exc_info()[1], self.parserline)) return self.top diff --git a/test.py b/test.py index b1039d11..bd021f4a 100644 --- a/test.py +++ b/test.py @@ -3,7 +3,7 @@ # test comment import datetime -#from token import * +from token import * from time import sleep from token import OP as OP_TEST, INDENT as INDENT_TEST @@ -139,10 +139,12 @@ class c1(): (c1().c2.\ c, 1, c3()) [0].pop() +c = u"asdf".join([1,2]) c = c1().c3().sleep() asdf = c1; asdf2 = asdf b= asdf2 #import parsing as test c = b().c3() -c = "a".join([1,2]) + +"1.0".join()