From edb401eb48d22535a0ad2e04b1e7508eca66c787 Mon Sep 17 00:00:00 2001 From: David Halter Date: Thu, 7 Feb 2013 14:06:14 +0100 Subject: [PATCH 01/83] basic implementation of recursive array parser in statement --- jedi/parsing_representation.py | 82 +++++++++++++++++++++++++++++----- 1 file changed, 71 insertions(+), 11 deletions(-) diff --git a/jedi/parsing_representation.py b/jedi/parsing_representation.py index b6a4646a..be6c005b 100644 --- a/jedi/parsing_representation.py +++ b/jedi/parsing_representation.py @@ -662,8 +662,9 @@ class Statement(Simple): '_assignment_calls_calculated') def __init__(self, module, code, set_vars, used_funcs, used_vars, - token_list, start_pos, end_pos): + token_list, start_pos, end_pos, parent=None): super(Statement, self).__init__(module, start_pos, end_pos) + # TODO remove code -> much cleaner self.code = code self.used_funcs = used_funcs self.used_vars = used_vars @@ -671,6 +672,7 @@ class Statement(Simple): for s in set_vars + used_funcs + used_vars: s.parent = self.use_as_parent self.set_vars = self._remove_executions_from_set_vars(set_vars) + self.parent = parent # cache self._assignment_calls = None @@ -741,6 +743,49 @@ class Statement(Simple): This is not really nice written, sorry for that. If you plan to replace it and make it nicer, that would be cool :-) """ + brackets = {'(': Array.TUPLE, '[': Array.LIST, '{': Array.SET} + closing_brackets = [')', '}', ']'] + + def parse_array(token_iterator, array_type, start_pos): + arr = Array(start_pos, array_type) + maybe_dict = array_type == Array.SET + while True: + statement, break_tok = parse_statement(token_iterator, + start_pos, maybe_dict) + if statement is not None: + is_key = maybe_dict and break_tok == ':' + arr.add_statement(statement, is_key) + return arr + + def parse_statement(token_iterator, start_pos, maybe_dict=False): + token_list = [] + level = 0 + tok = None + for i, tok_temp in token_iterator: + try: + token_type, tok, start_tok_pos = tok_temp + except TypeError: + # the token is a Name, which has already been parsed + tok = tok_temp + end_pos = tok.end_pos + else: + if tok in closing_brackets: + level -= 1 + elif tok in brackets.keys(): + level += 1 + + if level == 0 and tok in closing_brackets + (',',): + break + token_list.append(tok_temp) + + if not token_list: + return None, tok + + statement = Statement(self.module, "XXX" + self.code, [], [], [], + token_list, start_pos, end_pos) + statement.parent = self.parent + return statement + if self._assignment_calls_calculated: return self._assignment_calls self._assignment_details = [] @@ -749,8 +794,11 @@ class Statement(Simple): is_chain = False close_brackets = False - tok_iter = enumerate(self.token_list) - for i, tok_temp in tok_iter: + is_call = lambda: type(result) == Call + is_call_or_close = lambda: is_call() or close_brackets + + token_iterator = enumerate(self.token_list) + for i, tok_temp in token_iterator: #print 'tok', tok_temp, result if isinstance(tok_temp, ListComprehension): result.add_to_current_field(tok_temp) @@ -782,14 +830,18 @@ class Statement(Simple): is_chain = False continue elif tok == 'as': - next(tok_iter, None) + next(token_iterator, None) continue - # here starts the statement creation madness! - brackets = {'(': Array.TUPLE, '[': Array.LIST, '{': Array.SET} - is_call = lambda: type(result) == Call - is_call_or_close = lambda: is_call() or close_brackets + if level >= 0: + if tok in brackets.keys(): # brackets + level += 1 + elif tok in closing_brackets: + level -= 1 + if level == 0: + pass + # here starts the statement creation madness! is_literal = token_type in [tokenize.STRING, tokenize.NUMBER] if isinstance(tok, Name) or is_literal: _tok = tok @@ -847,7 +899,7 @@ class Statement(Simple): # important - it cannot be empty anymore if result.type == Array.NOARRAY: result.type = Array.TUPLE - elif tok in [')', '}', ']']: + elif tok in closing_brackets: while is_call_or_close(): result = result.parent close_brackets = False @@ -1021,7 +1073,7 @@ class Array(Call): below. :type array_type: int """ - NOARRAY = None + NOARRAY = None # just brackets, like `1 * (3 + 2)` TUPLE = 'tuple' LIST = 'list' DICT = 'dict' @@ -1056,7 +1108,15 @@ class Array(Call): one array. """ self.arr_el_pos.append(start_pos) - self.values.append([]) + self.statements.append([]) + + def add_statement(self, statement, is_key=False): + """Just add a new statement""" + statement.parent = self + if is_key: + self.keys.append(statement) + else: + self.values.append(statement) def add_to_current_field(self, tok): """ Adds a token to the latest field (in content). """ From d2ab0fe862bba9e74f3083f3f898674b00537481 Mon Sep 17 00:00:00 2001 From: David Halter Date: Thu, 7 Feb 2013 16:28:38 +0100 Subject: [PATCH 02/83] basic concept for get_assignment_calls --- jedi/parsing_representation.py | 142 ++++++++++----------------------- 1 file changed, 41 insertions(+), 101 deletions(-) diff --git a/jedi/parsing_representation.py b/jedi/parsing_representation.py index be6c005b..52972220 100644 --- a/jedi/parsing_representation.py +++ b/jedi/parsing_representation.py @@ -743,11 +743,17 @@ class Statement(Simple): This is not really nice written, sorry for that. If you plan to replace it and make it nicer, that would be cool :-) """ + if self._assignment_calls_calculated: + return self._assignment_calls + brackets = {'(': Array.TUPLE, '[': Array.LIST, '{': Array.SET} closing_brackets = [')', '}', ']'] - def parse_array(token_iterator, array_type, start_pos): + def parse_array(token_iterator, array_type, start_pos, add_el=None): arr = Array(start_pos, array_type) + if add_el is not None: + arr.add_statement(add_el) + maybe_dict = array_type == Array.SET while True: statement, break_tok = parse_statement(token_iterator, @@ -755,6 +761,10 @@ class Statement(Simple): if statement is not None: is_key = maybe_dict and break_tok == ':' arr.add_statement(statement, is_key) + if not arr.values and maybe_dict: + # this is a really special case - empty brackets {} are + # always dictionaries and not sets. + arr.type = Array.DICT return arr def parse_statement(token_iterator, start_pos, maybe_dict=False): @@ -764,6 +774,7 @@ class Statement(Simple): for i, tok_temp in token_iterator: try: token_type, tok, start_tok_pos = tok_temp + end_pos = start_pos[0], start_pos[1] + len(tok) except TypeError: # the token is a Name, which has already been parsed tok = tok_temp @@ -786,15 +797,12 @@ class Statement(Simple): statement.parent = self.parent return statement - if self._assignment_calls_calculated: - return self._assignment_calls self._assignment_details = [] - top = result = Array(self.start_pos, Array.NOARRAY, self) - level = 0 + result = [] is_chain = False close_brackets = False - is_call = lambda: type(result) == Call + is_call = lambda: result and type(result[-1]) == Call is_call_or_close = lambda: is_call() or close_brackets token_iterator = enumerate(self.token_list) @@ -810,41 +818,22 @@ class Statement(Simple): tok = tok_temp token_type = None start_pos = tok.start_pos - except ValueError: - debug.warning("unkown value, shouldn't happen", - tok_temp, type(tok_temp)) - raise else: - if level == 0 and tok.endswith('=') \ - and not tok in ['>=', '<=', '==', '!=']: + if tok.endswith('=') and not tok in ['>=', '<=', '==', '!=']: # This means, there is an assignment here. - # Add assignments, which can be more than one - self._assignment_details.append((tok, top)) - # All these calls wouldn't be important if nonlocal would - # exist. -> Initialize the first items again. - end_pos = start_pos[0], start_pos[1] + len(tok) - top = result = Array(end_pos, Array.NOARRAY, self) - level = 0 + self._assignment_details.append((tok, result)) + # nonlocal plz! + result = [] close_brackets = False is_chain = False continue - elif tok == 'as': + elif tok == 'as': # just ignore as next(token_iterator, None) continue - if level >= 0: - if tok in brackets.keys(): # brackets - level += 1 - elif tok in closing_brackets: - level -= 1 - if level == 0: - pass - - # here starts the statement creation madness! is_literal = token_type in [tokenize.STRING, tokenize.NUMBER] if isinstance(tok, Name) or is_literal: - _tok = tok c_type = Call.NAME if is_literal: tok = literal_eval(tok) @@ -853,87 +842,38 @@ class Statement(Simple): elif token_type == tokenize.NUMBER: c_type = Call.NUMBER + call = Call(tok, c_type, start_pos, parent=result) if is_chain: - call = Call(tok, c_type, start_pos, parent=result) - result = result.set_next_chain_call(call) + result[-1].set_next(call) is_chain = False close_brackets = False else: - if close_brackets: - result = result.parent - close_brackets = False - if type(result) == Call: - result = result.parent - call = Call(tok, c_type, start_pos, parent=result) - result.add_to_current_field(call) - result = call - tok = _tok - elif tok in brackets.keys(): # brackets - level += 1 + result.append(call) + elif tok in brackets.keys(): + arr = parse_array(token_iterator, brackets[tok], start_pos) if is_call_or_close(): - result = Array(start_pos, brackets[tok], parent=result) - result = result.parent.add_execution(result) - close_brackets = False + result[-1].add_execution(arr) else: - result = Array(start_pos, brackets[tok], parent=result) - result.parent.add_to_current_field(result) - elif tok == ':': - while is_call_or_close(): - result = result.parent - close_brackets = False - if result.type == Array.LIST: # [:] lookups - result.add_to_current_field(tok) - else: - result.add_dictionary_key() - elif tok == '.': - if close_brackets and result.parent != top: - # only get out of the array, if it is a array execution - result = result.parent - close_brackets = False - is_chain = True - elif tok == ',': - while is_call_or_close(): - result = result.parent - close_brackets = False - result.add_field((start_pos[0], start_pos[1] + 1)) - # important - it cannot be empty anymore - if result.type == Array.NOARRAY: - result.type = Array.TUPLE - elif tok in closing_brackets: - while is_call_or_close(): - result = result.parent - close_brackets = False - if tok == '}' and not len(result): - # this is a really special case - empty brackets {} are - # always dictionaries and not sets. - result.type = Array.DICT - level -= 1 - result.end_pos = start_pos[0], start_pos[1] + 1 + result.append(arr) close_brackets = True + elif tok == '.': + if result and isinstance(result[-1], Call): + is_chain = True + elif tok == ',': # implies a tuple + # rewrite `result`, because now the whole thing is a tuple + add_el = parse_statement(iter(result), start_pos) + arr = parse_array(token_iterator, Array.TUPLE, start_pos, + add_el) + result = [arr] else: - while is_call_or_close(): - result = result.parent - close_brackets = False + close_brackets = False if tok != '\n': - result.add_to_current_field(tok) - - if level != 0: - debug.warning("Brackets don't match: %s." - "This is not normal behaviour." % level) - - if self.token_list: - while result is not None: - try: - result.end_pos = start_pos[0], start_pos[1] + len(tok) - except TypeError: - result.end_pos = tok.end_pos - result = result.parent - else: - result.end_pos = self.end_pos + result.append(tok) + # TODO check self._assignment_calls_calculated = True - self._assignment_calls = top - return top + self._assignment_calls = result + return result class Param(Statement): @@ -1011,7 +951,7 @@ class Call(Base): def parent_stmt(self, value): self._parent_stmt = value - def set_next_chain_call(self, call): + def set_next(self, call): """ Adds another part of the statement""" self.next = call call.parent = self.parent From d6a5a3e0e7c38aaae555a2f0ad9f9895dd3fc0cd Mon Sep 17 00:00:00 2001 From: David Halter Date: Thu, 7 Feb 2013 17:39:40 +0100 Subject: [PATCH 03/83] further removals of unused code, also changes evaluate, now. --- jedi/evaluate.py | 80 ++++++++++---------- jedi/evaluate_representation.py | 1 - jedi/parsing_representation.py | 126 ++++++++++++++------------------ 3 files changed, 94 insertions(+), 113 deletions(-) diff --git a/jedi/evaluate.py b/jedi/evaluate.py index 3c8cb69a..427f20a5 100644 --- a/jedi/evaluate.py +++ b/jedi/evaluate.py @@ -580,52 +580,52 @@ def follow_call_list(call_list, follow_array=False): return loop if pr.Array.is_type(call_list, pr.Array.TUPLE, pr.Array.DICT): + raise NotImplementedError('TODO') # Tuples can stand just alone without any braces. These would be # recognized as separate calls, but actually are a tuple. result = follow_call(call_list) else: result = [] - for calls in call_list: - calls_iterator = iter(calls) - for call in calls_iterator: - if pr.Array.is_type(call, pr.Array.NOARRAY): - result += follow_call_list(call, follow_array=True) - elif isinstance(call, pr.ListComprehension): - loop = evaluate_list_comprehension(call) - stmt = copy.copy(call.stmt) - stmt.parent = loop - # create a for loop which does the same as list - # comprehensions - result += follow_statement(stmt) - else: - if isinstance(call, (pr.Lambda)): - result.append(er.Function(call)) - # With things like params, these can also be functions... - elif isinstance(call, (er.Function, er.Class, er.Instance, - dynamic.ArrayInstance)): - result.append(call) - # The string tokens are just operations (+, -, etc.) - elif not isinstance(call, (str, unicode)): - if str(call.name) == 'if': - # Ternary operators. - while True: - try: - call = next(calls_iterator) - except StopIteration: + calls_iterator = iter(call_list) + for call in calls_iterator: + if pr.Array.is_type(call, pr.Array.NOARRAY): + result += follow_call_list(call, follow_array=True) + elif isinstance(call, pr.ListComprehension): + loop = evaluate_list_comprehension(call) + stmt = copy.copy(call.stmt) + stmt.parent = loop + # create a for loop which does the same as list + # comprehensions + result += follow_statement(stmt) + else: + if isinstance(call, (pr.Lambda)): + result.append(er.Function(call)) + # With things like params, these can also be functions... + elif isinstance(call, (er.Function, er.Class, er.Instance, + dynamic.ArrayInstance)): + result.append(call) + # The string tokens are just operations (+, -, etc.) + elif not isinstance(call, (str, unicode)): + if str(call.name) == 'if': + # Ternary operators. + while True: + try: + call = next(calls_iterator) + except StopIteration: + break + try: + if str(call.name) == 'else': break - try: - if str(call.name) == 'else': - break - except AttributeError: - pass - continue - result += follow_call(call) - elif call == '*': - if [r for r in result if isinstance(r, er.Array) - or isinstance(r, er.Instance) - and str(r.name) == 'str']: - # if it is an iterable, ignore * operations - next(calls_iterator) + except AttributeError: + pass + continue + result += follow_call(call) + elif call == '*': + if [r for r in result if isinstance(r, er.Array) + or isinstance(r, er.Instance) + and str(r.name) == 'str']: + # if it is an iterable, ignore * operations + next(calls_iterator) if follow_array and isinstance(call_list, pr.Array): # call_list can also be a two dimensional array diff --git a/jedi/evaluate_representation.py b/jedi/evaluate_representation.py index 1f6b2815..b3171be4 100644 --- a/jedi/evaluate_representation.py +++ b/jedi/evaluate_representation.py @@ -497,7 +497,6 @@ class Execution(Executable): new_param = copy.copy(param) if parent_stmt is not None: new_param.parent = parent_stmt - new_param._assignment_calls_calculated = True new_param._assignment_calls = calls new_param.is_generated = True name = copy.copy(param.get_name()) diff --git a/jedi/parsing_representation.py b/jedi/parsing_representation.py index 52972220..96406f95 100644 --- a/jedi/parsing_representation.py +++ b/jedi/parsing_representation.py @@ -45,33 +45,11 @@ class Base(object): return isinstance(self, cls) -class Simple(Base): - """ - The super class for Scope, Import, Name and Statement. Every object in - the parser tree inherits from this class. - """ - __slots__ = ('parent', 'module', '_start_pos', 'use_as_parent', '_end_pos') - +class BasePosition(Base): def __init__(self, module, start_pos, end_pos=(None, None)): + self.module = module self._start_pos = start_pos self._end_pos = end_pos - self.parent = None - # use this attribute if parent should be something else than self. - self.use_as_parent = self - self.module = module - - @Python3Method - def get_parent_until(self, classes=(), reverse=False, - include_current=True): - """ Takes always the parent, until one class (not a Class) """ - if type(classes) not in (tuple, list): - classes = (classes,) - scope = self if include_current else self.parent - while scope.parent is not None: - if classes and reverse != scope.isinstance(*classes): - break - scope = scope.parent - return scope @property def start_pos(self): @@ -91,6 +69,33 @@ class Simple(Base): def end_pos(self, value): self._end_pos = value + +class Simple(BasePosition): + """ + The super class for Scope, Import, Name and Statement. Every object in + the parser tree inherits from this class. + """ + __slots__ = ('parent', 'module', '_start_pos', 'use_as_parent', '_end_pos') + + def __init__(self, module, start_pos, end_pos=(None, None)): + super(Simple, self).__init__(module, start_pos, end_pos) + self.parent = None + # use this attribute if parent should be something else than self. + self.use_as_parent = self + + @Python3Method + def get_parent_until(self, classes=(), reverse=False, + include_current=True): + """ Takes always the parent, until one class (not a Class) """ + if type(classes) not in (tuple, list): + classes = (classes,) + scope = self if include_current else self.parent + while scope.parent is not None: + if classes and reverse != scope.isinstance(*classes): + break + scope = scope.parent + return scope + def __repr__(self): code = self.get_code().replace('\n', ' ') return "<%s: %s@%s>" % \ @@ -658,8 +663,7 @@ class Statement(Simple): :type start_pos: tuple(int, int) """ __slots__ = ('used_funcs', 'code', 'token_list', 'used_vars', - 'set_vars', '_assignment_calls', '_assignment_details', - '_assignment_calls_calculated') + 'set_vars', '_assignment_calls', '_assignment_details') def __init__(self, module, code, set_vars, used_funcs, used_vars, token_list, start_pos, end_pos, parent=None): @@ -678,7 +682,6 @@ class Statement(Simple): self._assignment_calls = None self._assignment_details = None # this is important for other scripts - self._assignment_calls_calculated = False def _remove_executions_from_set_vars(self, set_vars): """ @@ -735,6 +738,13 @@ class Statement(Simple): return str(self.token_list[0]) == "global" def get_assignment_calls(self): + if self._assignment_calls is None: + # TODO check + result = self._parse_statement() + self._assignment_calls = result + return self._assignment_calls + + def _parse_statement(self): """ This is not done in the main parser, because it might be slow and most of the statements won't need this data anyway. This is something @@ -743,12 +753,6 @@ class Statement(Simple): This is not really nice written, sorry for that. If you plan to replace it and make it nicer, that would be cool :-) """ - if self._assignment_calls_calculated: - return self._assignment_calls - - brackets = {'(': Array.TUPLE, '[': Array.LIST, '{': Array.SET} - closing_brackets = [')', '}', ']'] - def parse_array(token_iterator, array_type, start_pos, add_el=None): arr = Array(start_pos, array_type) if add_el is not None: @@ -797,13 +801,13 @@ class Statement(Simple): statement.parent = self.parent return statement + # initializations self._assignment_details = [] result = [] is_chain = False close_brackets = False - - is_call = lambda: result and type(result[-1]) == Call - is_call_or_close = lambda: is_call() or close_brackets + brackets = {'(': Array.TUPLE, '[': Array.LIST, '{': Array.SET} + closing_brackets = [')', '}', ']'] token_iterator = enumerate(self.token_list) for i, tok_temp in token_iterator: @@ -842,24 +846,26 @@ class Statement(Simple): elif token_type == tokenize.NUMBER: c_type = Call.NUMBER - call = Call(tok, c_type, start_pos, parent=result) + call = Call(self.module, tok, c_type, start_pos, self) if is_chain: result[-1].set_next(call) - is_chain = False - close_brackets = False else: result.append(call) + is_chain = False + close_brackets = False elif tok in brackets.keys(): arr = parse_array(token_iterator, brackets[tok], start_pos) - if is_call_or_close(): + if type(result[-1]) == Call or close_brackets: result[-1].add_execution(arr) else: result.append(arr) close_brackets = True elif tok == '.': + close_brackets = False if result and isinstance(result[-1], Call): is_chain = True elif tok == ',': # implies a tuple + close_brackets = False # rewrite `result`, because now the whole thing is a tuple add_el = parse_statement(iter(result), start_pos) arr = parse_array(token_iterator, Array.TUPLE, start_pos, @@ -870,9 +876,6 @@ class Statement(Simple): if tok != '\n': result.append(tok) - # TODO check - self._assignment_calls_calculated = True - self._assignment_calls = result return result @@ -908,7 +911,7 @@ class Param(Statement): return n[0] -class Call(Base): +class Call(BasePosition): """ `Call` contains a call, e.g. `foo.bar` and owns the executions of those calls, which are `Array`s. @@ -917,45 +920,24 @@ class Call(Base): NUMBER = 2 STRING = 3 - def __init__(self, name, type, start_pos, parent_stmt=None, parent=None): + def __init__(self, module, name, type, start_pos, parent=None): + super(Call, self).__init__(module, start_pos) self.name = name # parent is not the oposite of next. The parent of c: a = [b.c] would # be an array. self.parent = parent self.type = type - self.start_pos = start_pos self.next = None self.execution = None - self._parent_stmt = parent_stmt - - @property - def start_pos(self): - offset = self.parent_stmt.module.line_offset - return offset + self._start_pos[0], self._start_pos[1] - - @start_pos.setter - def start_pos(self, value): - self._start_pos = value - - @property - def parent_stmt(self): - if self._parent_stmt is not None: - return self._parent_stmt - elif self.parent: - return self.parent.parent_stmt - else: - return None - - @parent_stmt.setter - def parent_stmt(self, value): - self._parent_stmt = value def set_next(self, call): """ Adds another part of the statement""" - self.next = call - call.parent = self.parent - return call + if self.next is not None: + self.next.set_next(call) + else: + self.next = call + call.parent = self.parent def add_execution(self, call): """ From f2451102c14afc966ad70d45f8c0a9353d282793 Mon Sep 17 00:00:00 2001 From: David Halter Date: Thu, 7 Feb 2013 17:58:44 +0100 Subject: [PATCH 04/83] end_pos crap of Array --- jedi/parsing_representation.py | 39 ++++++++++++++-------------------- 1 file changed, 16 insertions(+), 23 deletions(-) diff --git a/jedi/parsing_representation.py b/jedi/parsing_representation.py index 96406f95..167313e0 100644 --- a/jedi/parsing_representation.py +++ b/jedi/parsing_representation.py @@ -724,22 +724,19 @@ class Statement(Simple): """ Get the names for the statement. """ return list(self.set_vars) - @property - def assignment_details(self): - if self._assignment_details is None: - # normally, this calls sets this variable - self.get_assignment_calls() - # it may not have been set by get_assignment_calls -> just use an empty - # array - return self._assignment_details or [] - def is_global(self): # first keyword of the first token is global -> must be a global return str(self.token_list[0]) == "global" + @property + def assignment_details(self): + if self._assignment_calls is None: + # parse statement and therefore get the assignment details. + self._parse_statement() + return self._assignment_details + def get_assignment_calls(self): if self._assignment_calls is None: - # TODO check result = self._parse_statement() self._assignment_calls = result return self._assignment_calls @@ -759,6 +756,7 @@ class Statement(Simple): arr.add_statement(add_el) maybe_dict = array_type == Array.SET + break_tok = '' while True: statement, break_tok = parse_statement(token_iterator, start_pos, maybe_dict) @@ -769,6 +767,13 @@ class Statement(Simple): # this is a really special case - empty brackets {} are # always dictionaries and not sets. arr.type = Array.DICT + + arr.set_end_pos(start_pos, ) + k, v = arr.keys, arr.values + latest = (v[-1] if v else k[-1] if k else None) + end_pos = latest.end_pos if latest is not None \ + else start_pos[0], start_pos[1] + 1 + arr.end_pos = end_pos[0], end_pos[1] + len(break_tok) return arr def parse_statement(token_iterator, start_pos, maybe_dict=False): @@ -875,7 +880,6 @@ class Statement(Simple): close_brackets = False if tok != '\n': result.append(tok) - return result @@ -1009,18 +1013,7 @@ class Array(Call): self.values = values if values else [] self.arr_el_pos = [] self.keys = [] - self._end_pos = None, None - - @property - def end_pos(self): - if None in self._end_pos: - return self._end_pos - offset = self.parent_stmt.module.line_offset - return offset + self._end_pos[0], self._end_pos[1] - - @end_pos.setter - def end_pos(self, value): - self._end_pos = value + self.end_pos = None, None def add_field(self, start_pos): """ From 0eeb1036ac267c28e93d5886d369e80532e61730 Mon Sep 17 00:00:00 2001 From: David Halter Date: Thu, 7 Feb 2013 18:07:20 +0100 Subject: [PATCH 05/83] removed more unused methods from Array --- jedi/parsing_representation.py | 46 ++++------------------------------ 1 file changed, 5 insertions(+), 41 deletions(-) diff --git a/jedi/parsing_representation.py b/jedi/parsing_representation.py index 167313e0..ff2d04af 100644 --- a/jedi/parsing_representation.py +++ b/jedi/parsing_representation.py @@ -817,9 +817,6 @@ class Statement(Simple): token_iterator = enumerate(self.token_list) for i, tok_temp in token_iterator: #print 'tok', tok_temp, result - if isinstance(tok_temp, ListComprehension): - result.add_to_current_field(tok_temp) - continue try: token_type, tok, start_pos = tok_temp except TypeError: @@ -1009,22 +1006,10 @@ class Array(Call): parent=None, values=None): super(Array, self).__init__(None, arr_type, start_pos, parent_stmt, parent) - self.values = values if values else [] - self.arr_el_pos = [] self.keys = [] self.end_pos = None, None - def add_field(self, start_pos): - """ - Just add a new field to the values. - - Each value has a sub-array, because there may be different tokens in - one array. - """ - self.arr_el_pos.append(start_pos) - self.statements.append([]) - def add_statement(self, statement, is_key=False): """Just add a new statement""" statement.parent = self @@ -1033,29 +1018,6 @@ class Array(Call): else: self.values.append(statement) - def add_to_current_field(self, tok): - """ Adds a token to the latest field (in content). """ - if not self.values: - # An empty round brace is just a tuple, filled it is unnecessary. - if self.type == Array.TUPLE: - self.type = Array.NOARRAY - # Add the first field, this is done here, because if nothing - # gets added, the list is empty, which is also needed sometimes. - self.values.append([]) - self.values[-1].append(tok) - - def add_dictionary_key(self): - """ - Only used for dictionaries, automatically adds the tokens added by now - from the values to keys, because the parser works this way. - """ - if self.type in (Array.LIST, Array.TUPLE): - return # these are basically code errors, just ignore - self.keys.append(self.values.pop()) - if self.type == Array.SET: - self.type = Array.DICT - self.values.append([]) - def get_only_subelement(self): """ Returns the only element that an array contains. If it contains @@ -1070,6 +1032,7 @@ class Array(Call): """ This is not only used for calls on the actual object, but for ducktyping, to invoke this function with anything as `self`. + TODO remove? """ if isinstance(instance, Array): if instance.type in types: @@ -1080,13 +1043,14 @@ class Array(Call): return len(self.values) def __getitem__(self, key): + if self.type == self.DICT: + raise NotImplementedError('no dicts allowed, yet') return self.values[key] def __iter__(self): if self.type == self.DICT: - return iter(zip(self.keys, self.values)) - else: - return iter(self.values) + raise NotImplementedError('no dicts allowed, yet') + return iter(self.values) def get_code(self): def to_str(el): From 6d2d23cd78d77bc5b9888b9eb9d67ccdb4ec95d4 Mon Sep 17 00:00:00 2001 From: David Halter Date: Thu, 7 Feb 2013 18:19:22 +0100 Subject: [PATCH 06/83] docstring update for parsing_representation --- jedi/parsing_representation.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/jedi/parsing_representation.py b/jedi/parsing_representation.py index ff2d04af..2b2757ed 100644 --- a/jedi/parsing_representation.py +++ b/jedi/parsing_representation.py @@ -8,13 +8,12 @@ inherited. It's used by ``Function``, ``Class``, ``Flow``, etc. A ``Scope`` may have ``subscopes``, ``imports`` and ``statements``. The entire parser is based on scopes, because they also stand for indentation. -One strange thing about the parser is that ``Array`` is two dimensional. This -has been caused by the fact that each array element can be defined by -operations: ``[1, 2+33]``. So I chose to use a second dimension for ``2+33``, -where each element would lie in the array like this: ``[2, '+', 33]``. In the -future it might be useful to use Statements there, too (remove those crappy two -dimensional arrays). This is also how ``Param`` works. Every single ``Param`` -is a ``Statement``. +One special thing: + +``Array`` values are statements. But if you think about it, this makes sense. +``[1, 2+33]`` for example would be an Array with two ``Statement`` inside. This +is the easiest way to write a parser. The same behaviour applies to ``Param``, +which is being used in a function definition. .. todo:: remove docstr params from Scope.__init__() From bf298f0ef4ff1877e3fae26b17973b23198bfb24 Mon Sep 17 00:00:00 2001 From: David Halter Date: Thu, 7 Feb 2013 18:34:18 +0100 Subject: [PATCH 07/83] start to get rid of parent_stmt --- jedi/evaluate.py | 5 ++--- jedi/parsing_representation.py | 36 ++++++++++++++-------------------- 2 files changed, 17 insertions(+), 24 deletions(-) diff --git a/jedi/evaluate.py b/jedi/evaluate.py index 427f20a5..a7a12229 100644 --- a/jedi/evaluate.py +++ b/jedi/evaluate.py @@ -639,10 +639,9 @@ def follow_call_list(call_list, follow_array=False): def follow_call(call): """Follow a call is following a function, variable, string, etc.""" - scope = call.parent_stmt.parent path = call.generate_call_path() - position = call.parent_stmt.start_pos - return follow_call_path(path, scope, position) + scope = call.get_parent_until(pr.Scope) + return follow_call_path(path, scope, call.start_pos) def follow_call_path(path, scope, position): diff --git a/jedi/parsing_representation.py b/jedi/parsing_representation.py index 2b2757ed..4bd77d1c 100644 --- a/jedi/parsing_representation.py +++ b/jedi/parsing_representation.py @@ -44,12 +44,22 @@ class Base(object): return isinstance(self, cls) -class BasePosition(Base): +class Simple(Base): + """ + The super class for Scope, Import, Name and Statement. Every object in + the parser tree inherits from this class. + """ + __slots__ = ('parent', 'module', '_start_pos', 'use_as_parent', '_end_pos') + def __init__(self, module, start_pos, end_pos=(None, None)): self.module = module self._start_pos = start_pos self._end_pos = end_pos + self.parent = None + # use this attribute if parent should be something else than self. + self.use_as_parent = self + @property def start_pos(self): return self.module.line_offset + self._start_pos[0], self._start_pos[1] @@ -68,20 +78,6 @@ class BasePosition(Base): def end_pos(self, value): self._end_pos = value - -class Simple(BasePosition): - """ - The super class for Scope, Import, Name and Statement. Every object in - the parser tree inherits from this class. - """ - __slots__ = ('parent', 'module', '_start_pos', 'use_as_parent', '_end_pos') - - def __init__(self, module, start_pos, end_pos=(None, None)): - super(Simple, self).__init__(module, start_pos, end_pos) - self.parent = None - # use this attribute if parent should be something else than self. - self.use_as_parent = self - @Python3Method def get_parent_until(self, classes=(), reverse=False, include_current=True): @@ -811,7 +807,7 @@ class Statement(Simple): is_chain = False close_brackets = False brackets = {'(': Array.TUPLE, '[': Array.LIST, '{': Array.SET} - closing_brackets = [')', '}', ']'] + closing_brackets = ')', '}', ']' token_iterator = enumerate(self.token_list) for i, tok_temp in token_iterator: @@ -911,7 +907,7 @@ class Param(Statement): return n[0] -class Call(BasePosition): +class Call(Simple): """ `Call` contains a call, e.g. `foo.bar` and owns the executions of those calls, which are `Array`s. @@ -1001,10 +997,8 @@ class Array(Call): DICT = 'dict' SET = 'set' - def __init__(self, start_pos, arr_type=NOARRAY, parent_stmt=None, - parent=None, values=None): - super(Array, self).__init__(None, arr_type, start_pos, parent_stmt, - parent) + def __init__(self, start_pos, arr_type=NOARRAY, parent=None, values=None): + super(Array, self).__init__(None, arr_type, start_pos, parent) self.values = values if values else [] self.keys = [] self.end_pos = None, None From 80d4c6e884511bb829bc6a1c1640680b18bd602f Mon Sep 17 00:00:00 2001 From: David Halter Date: Fri, 8 Feb 2013 01:36:58 +0100 Subject: [PATCH 08/83] many refactoring fixes --- jedi/evaluate_representation.py | 10 ++++--- jedi/parsing_representation.py | 50 ++++++++++++++------------------- 2 files changed, 27 insertions(+), 33 deletions(-) diff --git a/jedi/evaluate_representation.py b/jedi/evaluate_representation.py index b3171be4..d824d2b2 100644 --- a/jedi/evaluate_representation.py +++ b/jedi/evaluate_representation.py @@ -11,6 +11,7 @@ they change classes in Python 3. """ import sys import copy +import itertools from _compatibility import property, use_metaclass, next, hasattr import parsing_representation as pr @@ -786,7 +787,7 @@ class Array(use_metaclass(cache.CachedMetaClass, pr.Base)): except (IndexError, KeyError): pass - result = list(self.follow_values(self._array.values)) + result = list(self._follow_values(self._array.values)) result += dynamic.check_array_additions(self) return set(result) @@ -811,11 +812,12 @@ class Array(use_metaclass(cache.CachedMetaClass, pr.Base)): if index is None: raise KeyError('No key found in dictionary') values = [self._array[index]] - return self.follow_values(values) + return self._follow_values(values) - def follow_values(self, values): + def _follow_values(self, values): """ helper function for the index getters """ - return evaluate.follow_call_list(values) + return itertools.chain.from_iterable(evaluate.follow_statement(v) + for v in values) def get_defined_names(self): """ diff --git a/jedi/parsing_representation.py b/jedi/parsing_representation.py index 4bd77d1c..c7b3e269 100644 --- a/jedi/parsing_representation.py +++ b/jedi/parsing_representation.py @@ -746,32 +746,34 @@ class Statement(Simple): it and make it nicer, that would be cool :-) """ def parse_array(token_iterator, array_type, start_pos, add_el=None): - arr = Array(start_pos, array_type) + arr = Array(self.module, start_pos, array_type) if add_el is not None: arr.add_statement(add_el) maybe_dict = array_type == Array.SET break_tok = '' while True: - statement, break_tok = parse_statement(token_iterator, + stmt, break_tok = parse_array_el(token_iterator, start_pos, maybe_dict) - if statement is not None: + if stmt is None: + break + else: is_key = maybe_dict and break_tok == ':' - arr.add_statement(statement, is_key) + arr.add_statement(stmt, is_key) if not arr.values and maybe_dict: # this is a really special case - empty brackets {} are # always dictionaries and not sets. arr.type = Array.DICT - arr.set_end_pos(start_pos, ) k, v = arr.keys, arr.values latest = (v[-1] if v else k[-1] if k else None) end_pos = latest.end_pos if latest is not None \ else start_pos[0], start_pos[1] + 1 - arr.end_pos = end_pos[0], end_pos[1] + len(break_tok) + arr.end_pos = end_pos[0], end_pos[1] + (len(break_tok) if break_tok + else 0) return arr - def parse_statement(token_iterator, start_pos, maybe_dict=False): + def parse_array_el(token_iterator, start_pos, maybe_dict=False): token_list = [] level = 0 tok = None @@ -799,7 +801,7 @@ class Statement(Simple): statement = Statement(self.module, "XXX" + self.code, [], [], [], token_list, start_pos, end_pos) statement.parent = self.parent - return statement + return statement, tok # initializations self._assignment_details = [] @@ -852,8 +854,8 @@ class Statement(Simple): close_brackets = False elif tok in brackets.keys(): arr = parse_array(token_iterator, brackets[tok], start_pos) - if type(result[-1]) == Call or close_brackets: - result[-1].add_execution(arr) + if result and (type(result[-1]) == Call or close_brackets): + result[-1].set_execution(arr) else: result.append(arr) close_brackets = True @@ -864,7 +866,7 @@ class Statement(Simple): elif tok == ',': # implies a tuple close_brackets = False # rewrite `result`, because now the whole thing is a tuple - add_el = parse_statement(iter(result), start_pos) + add_el = parse_array_el(iter(result), start_pos) arr = parse_array(token_iterator, Array.TUPLE, start_pos, add_el) result = [arr] @@ -935,19 +937,18 @@ class Call(Simple): self.next = call call.parent = self.parent - def add_execution(self, call): + def set_execution(self, call): """ An execution is nothing else than brackets, with params in them, which shows access on the internals of this name. """ - self.execution = call - # there might be multiple executions, like a()[0], in that case, they - # have the same parent. Otherwise it's not possible to parse proper. - if self.parent.execution == self: - call.parent = self.parent + if self.next is not None: + self.next.set_execution(call) + elif self.execution is not None: + self.execution.set_execution(call) else: + self.execution = call call.parent = self - return call def generate_call_path(self): """ Helps to get the order in which statements are executed. """ @@ -997,8 +998,8 @@ class Array(Call): DICT = 'dict' SET = 'set' - def __init__(self, start_pos, arr_type=NOARRAY, parent=None, values=None): - super(Array, self).__init__(None, arr_type, start_pos, parent) + def __init__(self, module, start_pos, arr_type=NOARRAY, parent=None, values=None): + super(Array, self).__init__(module, None, arr_type, start_pos, parent) self.values = values if values else [] self.keys = [] self.end_pos = None, None @@ -1011,15 +1012,6 @@ class Array(Call): else: self.values.append(statement) - def get_only_subelement(self): - """ - Returns the only element that an array contains. If it contains - more than one element, raise an exception. - """ - if len(self.values) != 1 or len(self.values[0]) != 1: - raise AttributeError("More than one value found") - return self.values[0][0] - @staticmethod def is_type(instance, *types): """ From 5737e3ad1d538207adebbf0eaed8b8932241a188 Mon Sep 17 00:00:00 2001 From: David Halter Date: Fri, 8 Feb 2013 02:22:02 +0100 Subject: [PATCH 09/83] fix __repr__ functions --- jedi/parsing_representation.py | 39 +++++++++++++++++++--------------- 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/jedi/parsing_representation.py b/jedi/parsing_representation.py index c7b3e269..7cf3e965 100644 --- a/jedi/parsing_representation.py +++ b/jedi/parsing_representation.py @@ -710,10 +710,17 @@ class Statement(Simple): return list(result) def get_code(self, new_line=True): + code = '' + for c in self.get_assignment_calls(): + if isinstance(c, Call): + code += c.get_code() + else: + code += c + if new_line: - return self.code + '\n' + return code + '\n' else: - return self.code + return code def get_set_vars(self): """ Get the names for the statement. """ @@ -760,6 +767,8 @@ class Statement(Simple): else: is_key = maybe_dict and break_tok == ':' arr.add_statement(stmt, is_key) + if break_tok in closing_brackets: + break if not arr.values and maybe_dict: # this is a really special case - empty brackets {} are # always dictionaries and not sets. @@ -775,7 +784,7 @@ class Statement(Simple): def parse_array_el(token_iterator, start_pos, maybe_dict=False): token_list = [] - level = 0 + level = 1 tok = None for i, tok_temp in token_iterator: try: @@ -791,7 +800,7 @@ class Statement(Simple): elif tok in brackets.keys(): level += 1 - if level == 0 and tok in closing_brackets + (',',): + if level == 0 and tok in closing_brackets or level == 1 and tok == ',': break token_list.append(tok_temp) @@ -854,10 +863,14 @@ class Statement(Simple): close_brackets = False elif tok in brackets.keys(): arr = parse_array(token_iterator, brackets[tok], start_pos) - if result and (type(result[-1]) == Call or close_brackets): + if result and (isinstance(result[-1], Call) or close_brackets): + print 'x', arr result[-1].set_execution(arr) else: + print arr + arr.parent = self result.append(arr) + #print(tok, result) close_brackets = True elif tok == '.': close_brackets = False @@ -866,7 +879,7 @@ class Statement(Simple): elif tok == ',': # implies a tuple close_brackets = False # rewrite `result`, because now the whole thing is a tuple - add_el = parse_array_el(iter(result), start_pos) + add_el, t = parse_array_el(enumerate(result), start_pos) arr = parse_array(token_iterator, Array.TUPLE, start_pos, add_el) result = [arr] @@ -1038,12 +1051,6 @@ class Array(Call): return iter(self.values) def get_code(self): - def to_str(el): - try: - return el.get_code() - except AttributeError: - return str(el) - map = {Array.NOARRAY: '%s', Array.TUPLE: '(%s)', Array.LIST: '[%s]', @@ -1051,17 +1058,15 @@ class Array(Call): Array.SET: '{%s}' } inner = [] - for i, value in enumerate(self.values): + for i, stmt in enumerate(self.values): s = '' try: key = self.keys[i] except IndexError: pass else: - for el in key[i]: - s += to_str(el) - for el in value: - s += to_str(el) + s += key.get_code(new_line=False) + ': ' + s += stmt.get_code(new_line=False) inner.append(s) return map[self.type] % ', '.join(inner) From d82b8d5e190becf3d61d05a8276a590b2acf0e08 Mon Sep 17 00:00:00 2001 From: David Halter Date: Fri, 8 Feb 2013 02:43:05 +0100 Subject: [PATCH 10/83] remove close_brackets - not needed anymore --- jedi/evaluate_representation.py | 6 +++--- jedi/parsing_representation.py | 12 +----------- 2 files changed, 4 insertions(+), 14 deletions(-) diff --git a/jedi/evaluate_representation.py b/jedi/evaluate_representation.py index d824d2b2..6ba4effc 100644 --- a/jedi/evaluate_representation.py +++ b/jedi/evaluate_representation.py @@ -767,11 +767,11 @@ class Array(use_metaclass(cache.CachedMetaClass, pr.Base)): """ Get the types of a specific index or all, if not given """ # array slicing if index_call_list is not None: - if index_call_list and [x for x in index_call_list if ':' in x]: + print index_call_list + if index_call_list and [x for x in index_call_list if ':' in x.token_list]: return [self] - index_possibilities = list(evaluate.follow_call_list( - index_call_list)) + index_possibilities = [evaluate.follow_statement(i) for i in index_call_list] if len(index_possibilities) == 1: # This is indexing only one element, with a fixed index number, # otherwise it just ignores the index (e.g. [1+1]). diff --git a/jedi/parsing_representation.py b/jedi/parsing_representation.py index 7cf3e965..f9a0bb66 100644 --- a/jedi/parsing_representation.py +++ b/jedi/parsing_representation.py @@ -816,7 +816,6 @@ class Statement(Simple): self._assignment_details = [] result = [] is_chain = False - close_brackets = False brackets = {'(': Array.TUPLE, '[': Array.LIST, '{': Array.SET} closing_brackets = ')', '}', ']' @@ -835,9 +834,7 @@ class Statement(Simple): # This means, there is an assignment here. # Add assignments, which can be more than one self._assignment_details.append((tok, result)) - # nonlocal plz! result = [] - close_brackets = False is_chain = False continue elif tok == 'as': # just ignore as @@ -860,31 +857,24 @@ class Statement(Simple): else: result.append(call) is_chain = False - close_brackets = False elif tok in brackets.keys(): arr = parse_array(token_iterator, brackets[tok], start_pos) - if result and (isinstance(result[-1], Call) or close_brackets): - print 'x', arr + if result and isinstance(result[-1], Call): result[-1].set_execution(arr) else: - print arr arr.parent = self result.append(arr) #print(tok, result) - close_brackets = True elif tok == '.': - close_brackets = False if result and isinstance(result[-1], Call): is_chain = True elif tok == ',': # implies a tuple - close_brackets = False # rewrite `result`, because now the whole thing is a tuple add_el, t = parse_array_el(enumerate(result), start_pos) arr = parse_array(token_iterator, Array.TUPLE, start_pos, add_el) result = [arr] else: - close_brackets = False if tok != '\n': result.append(tok) return result From 94126edda89ffc975bb254d615490b75c2827d34 Mon Sep 17 00:00:00 2001 From: David Halter Date: Fri, 8 Feb 2013 12:32:19 +0100 Subject: [PATCH 11/83] get rid of generate_param_array --- jedi/evaluate.py | 9 ++--- jedi/evaluate_representation.py | 70 ++++++++++++++++++--------------- jedi/helpers.py | 13 ------ 3 files changed, 42 insertions(+), 50 deletions(-) diff --git a/jedi/evaluate.py b/jedi/evaluate.py index a7a12229..7cc4062c 100644 --- a/jedi/evaluate.py +++ b/jedi/evaluate.py @@ -82,7 +82,6 @@ import debug import builtin import imports import recursion -import helpers import dynamic import docstrings @@ -416,9 +415,8 @@ def check_getattr(inst, name_str): result = [] # str is important to lose the NamePart! name = pr.Call(str(name_str), pr.Call.STRING, (0, 0), inst) - args = helpers.generate_param_array([name]) try: - result = inst.execute_subscope_by_name('__getattr__', args) + result = inst.execute_subscope_by_name('__getattr__', [name]) except KeyError: pass if not result: @@ -427,7 +425,7 @@ def check_getattr(inst, name_str): # could be practical and the jedi would return wrong types. If # you ever have something, let me know! try: - result = inst.execute_subscope_by_name('__getattribute__', args) + result = inst.execute_subscope_by_name('__getattribute__', [name]) except KeyError: pass return result @@ -663,8 +661,7 @@ def follow_call_path(path, scope, position): debug.warning('unknown type:', current.type, current) scopes = [] # Make instances of those number/string objects. - arr = helpers.generate_param_array([current.name]) - scopes = [er.Instance(s, arr) for s in scopes] + scopes = [er.Instance(s, [current.name]) for s in scopes] result = imports.strip_imports(scopes) return follow_paths(path, result, scope, position=position) diff --git a/jedi/evaluate_representation.py b/jedi/evaluate_representation.py index 6ba4effc..95b5acbe 100644 --- a/jedi/evaluate_representation.py +++ b/jedi/evaluate_representation.py @@ -35,12 +35,12 @@ class DecoratorNotFound(LookupError): class Executable(pr.Base): - """ An instance is also an executable - because __init__ is called """ - def __init__(self, base, var_args=None): + """ + An instance is also an executable - because __init__ is called + :param var_args: The param input array, consist of `pr.Array` or list. + """ + def __init__(self, base, var_args=[]): self.base = base - # The param input array. - if var_args is None: - var_args = pr.Array(None, None) self.var_args = var_args def get_parent_until(self, *args, **kwargs): @@ -122,20 +122,15 @@ class Instance(use_metaclass(cache.CachedMetaClass, Executable)): sub = self.base.get_subscope_by_name(name) return InstanceElement(self, sub, True) - def execute_subscope_by_name(self, name, args=None): - if args is None: - args = helpers.generate_param_array([]) + def execute_subscope_by_name(self, name, args=[]): method = self.get_subscope_by_name(name) - if args.parent_stmt is None: - args.parent_stmt = method return Execution(method, args).get_return_types() def get_descriptor_return(self, obj): """ Throws a KeyError if there's no method. """ # Arguments in __get__ descriptors are obj, class. # `method` is the new parent of the array, don't know if that's good. - v = [obj, obj.base] if isinstance(obj, Instance) else [None, obj] - args = helpers.generate_param_array(v) + args = [obj, obj.base] if isinstance(obj, Instance) else [None, obj] return self.execute_subscope_by_name('__get__', args) @cache.memoize_default([]) @@ -165,7 +160,7 @@ class Instance(use_metaclass(cache.CachedMetaClass, Executable)): yield self, names def get_index_types(self, index=None): - args = helpers.generate_param_array([] if index is None else [index]) + args = [] if index is None else [index] try: return self.execute_subscope_by_name('__getitem__', args) except KeyError: @@ -340,9 +335,8 @@ class Function(use_metaclass(cache.CachedMetaClass, pr.Base)): decorator = dec_results.pop() # Create param array. old_func = Function(f, is_decorated=True) - params = helpers.generate_param_array([old_func], old_func) - wrappers = Execution(decorator, params).get_return_types() + wrappers = Execution(decorator, [old_func]).get_return_types() if not len(wrappers): debug.warning('no wrappers found', self.base_func) return None @@ -389,6 +383,17 @@ class Execution(Executable): multiple calls to functions and recursion has to be avoided. But this is responsibility of the decorators. """ + def follow_var_arg(self, index): + try: + stmt = self.var_args[index] + except IndexError: + return [] + else: + if isinstance(stmt, pr.Statement): + return evaluate.follow_statement(stmt) + else: + return [stmt] # just some arbitrary object + @cache.memoize_default(default=[]) @recursion.ExecutionRecursionDecorator def get_return_types(self, evaluate_generator=False): @@ -402,8 +407,8 @@ class Execution(Executable): if func_name == 'getattr': # follow the first param try: - objects = evaluate.follow_call_list([self.var_args[0]]) - names = evaluate.follow_call_list([self.var_args[1]]) + objects = self.follow_var_arg(0) + names = self.follow_var_arg(1) except IndexError: debug.warning('getattr() called with to few args.') return [] @@ -421,11 +426,12 @@ class Execution(Executable): elif func_name == 'type': # otherwise it would be a metaclass if len(self.var_args) == 1: - objects = evaluate.follow_call_list([self.var_args[0]]) + objects = self.follow_var_arg(0) return [o.base for o in objects if isinstance(o, Instance)] elif func_name == 'super': + # TODO make this able to detect multiple inheritance supers accept = (pr.Function,) - func = self.var_args.parent_stmt.get_parent_until(accept) + func = self.var_args.get_parent_until(accept) if func.isinstance(*accept): cls = func.get_parent_until(accept + (pr.Class,), include_current=False) @@ -597,20 +603,20 @@ class Execution(Executable): """ def iterate(): # `var_args` is typically an Array, and not a list. - for var_arg in self.var_args: - # empty var_arg - if len(var_arg) == 0: - yield None, None + for stmt in self.var_args: + if not isinstance(stmt, pr.Statement): + yield None, stmt # *args - elif var_arg[0] == '*': - arrays = evaluate.follow_call_list([var_arg[1:]]) + elif stmt.token_list[0] == '*': + arrays = evaluate.follow_call_list([stmt.token_list[1:]]) + # *args must be some sort of an array, otherwise -> ignore for array in arrays: if hasattr(array, 'get_contents'): for field in array.get_contents(): yield None, field # **kwargs - elif var_arg[0] == '**': - arrays = evaluate.follow_call_list([var_arg[1:]]) + elif stmt[0] == '**': + arrays = evaluate.follow_call_list([stmt.token_list[1:]]) for array in arrays: if hasattr(array, 'get_contents'): for key, field in array.get_contents(): @@ -623,11 +629,13 @@ class Execution(Executable): yield name, field # Normal arguments (including key arguments). else: - if len(var_arg) > 1 and var_arg[1] == '=': - # This is a named parameter (var_arg[0] is a Call). - yield var_arg[0].name, var_arg[2:] + if stmt.assignment_detail: + tok, key_arr = stmt.assignment_detail[0] + # named parameter + if key_arr and isinstance(key_arr[0], pr.Call): + yield tok[0].name, stmt else: - yield None, var_arg + yield None, stmt return iter(common.PushBackIterator(iterate())) diff --git a/jedi/helpers.py b/jedi/helpers.py index b276a5dd..4478ba20 100644 --- a/jedi/helpers.py +++ b/jedi/helpers.py @@ -60,19 +60,6 @@ def fast_parent_copy(obj): return recursion(obj) -def generate_param_array(args_tuple, parent_stmt=None): - """ This generates an array, that can be used as a param. """ - values = [] - for arg in args_tuple: - if arg is None: - values.append([]) - else: - values.append([arg]) - pos = None - arr = pr.Array(pos, pr.Array.TUPLE, parent_stmt, values=values) - return arr - - def check_arr_index(arr, pos): positions = arr.arr_el_pos for index, comma_pos in enumerate(positions): From 39f43c52fe7c089ba5d6eca5c92b8430849e7ba3 Mon Sep 17 00:00:00 2001 From: David Halter Date: Fri, 8 Feb 2013 12:49:36 +0100 Subject: [PATCH 12/83] little fixes for the dynamic module --- jedi/dynamic.py | 8 ++++---- jedi/evaluate.py | 2 +- jedi/evaluate_representation.py | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/jedi/dynamic.py b/jedi/dynamic.py index 7e8034d5..7bc507c8 100644 --- a/jedi/dynamic.py +++ b/jedi/dynamic.py @@ -182,12 +182,12 @@ def search_params(param): def check_array_additions(array): """ Just a mapper function for the internal _check_array_additions """ - if array._array.type not in ['list', 'set']: + if not pr.Array.is_type(array._array, pr.Array.LIST, pr.Array.SET): # TODO also check for dict updates return [] is_list = array._array.type == 'list' - current_module = array._array.parent_stmt.get_parent_until() + current_module = array._array.get_parent_until() res = _check_array_additions(array, current_module, is_list) return res @@ -264,10 +264,10 @@ def _check_array_additions(compare_array, module, is_list): def get_execution_parent(element, *stop_classes): """ Used to get an Instance/Execution parent """ if isinstance(element, er.Array): - stmt = element._array.parent_stmt + stmt = element._array.parent else: # must be instance - stmt = element.var_args.parent_stmt + stmt = element.var_args.parent if isinstance(stmt, er.InstanceElement): stop_classes = list(stop_classes) + [er.Function] return stmt.get_parent_until(stop_classes) diff --git a/jedi/evaluate.py b/jedi/evaluate.py index 7cc4062c..7a6bc069 100644 --- a/jedi/evaluate.py +++ b/jedi/evaluate.py @@ -661,7 +661,7 @@ def follow_call_path(path, scope, position): debug.warning('unknown type:', current.type, current) scopes = [] # Make instances of those number/string objects. - scopes = [er.Instance(s, [current.name]) for s in scopes] + scopes = [er.Instance(s, (current.name,)) for s in scopes] result = imports.strip_imports(scopes) return follow_paths(path, result, scope, position=position) diff --git a/jedi/evaluate_representation.py b/jedi/evaluate_representation.py index 95b5acbe..6da69271 100644 --- a/jedi/evaluate_representation.py +++ b/jedi/evaluate_representation.py @@ -53,7 +53,7 @@ class Executable(pr.Base): class Instance(use_metaclass(cache.CachedMetaClass, Executable)): """ This class is used to evaluate instances. """ - def __init__(self, base, var_args=None): + def __init__(self, base, var_args=[]): super(Instance, self).__init__(base, var_args) if str(base.name) in ['list', 'set'] \ and builtin.Builtin.scope == base.get_parent_until(): @@ -336,7 +336,7 @@ class Function(use_metaclass(cache.CachedMetaClass, pr.Base)): # Create param array. old_func = Function(f, is_decorated=True) - wrappers = Execution(decorator, [old_func]).get_return_types() + wrappers = Execution(decorator, (old_func,)).get_return_types() if not len(wrappers): debug.warning('no wrappers found', self.base_func) return None From 4f060ddcd940b013e87b4d8aa3908187cd2422aa Mon Sep 17 00:00:00 2001 From: David Halter Date: Fri, 8 Feb 2013 13:09:08 +0100 Subject: [PATCH 13/83] refactor a lot of er.Array --- jedi/evaluate_representation.py | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/jedi/evaluate_representation.py b/jedi/evaluate_representation.py index 6da69271..172b91e6 100644 --- a/jedi/evaluate_representation.py +++ b/jedi/evaluate_representation.py @@ -775,24 +775,20 @@ class Array(use_metaclass(cache.CachedMetaClass, pr.Base)): """ Get the types of a specific index or all, if not given """ # array slicing if index_call_list is not None: - print index_call_list if index_call_list and [x for x in index_call_list if ':' in x.token_list]: return [self] - index_possibilities = [evaluate.follow_statement(i) for i in index_call_list] + index_possibilities = self._follow_values(index_call_list) if len(index_possibilities) == 1: # This is indexing only one element, with a fixed index number, # otherwise it just ignores the index (e.g. [1+1]). - try: - # Multiple elements in the array are not wanted. var_args - # and get_only_subelement can raise AttributeErrors. - i = index_possibilities[0].var_args.get_only_subelement() - except AttributeError: - pass - else: + index = index_possibilities[0] + if isinstance(index, Instance) \ + and str(index.name) in ['int', 'str'] \ + and len(index.var_args) == 1: try: - return self.get_exact_index_types(i) - except (IndexError, KeyError): + return self.get_exact_index_types(index.var_args[0]) + except (KeyError, IndexError): pass result = list(self._follow_values(self._array.values)) @@ -824,8 +820,8 @@ class Array(use_metaclass(cache.CachedMetaClass, pr.Base)): def _follow_values(self, values): """ helper function for the index getters """ - return itertools.chain.from_iterable(evaluate.follow_statement(v) - for v in values) + return list(itertools.chain.from_iterable(evaluate.follow_statement(v) + for v in values)) def get_defined_names(self): """ From 6c0a0e889c52d270887fff03ad5aeb62f72a2086 Mon Sep 17 00:00:00 2001 From: David Halter Date: Fri, 8 Feb 2013 15:14:43 +0100 Subject: [PATCH 14/83] fix start_pos of multiple elements --- jedi/api.py | 4 ++-- jedi/evaluate.py | 3 ++- jedi/parsing_representation.py | 16 +++++++++++----- jedi/recursion.py | 2 +- 4 files changed, 16 insertions(+), 9 deletions(-) diff --git a/jedi/api.py b/jedi/api.py index 017a3056..44718749 100644 --- a/jedi/api.py +++ b/jedi/api.py @@ -190,12 +190,12 @@ class Script(object): return scopes def _get_under_cursor_stmt(self, cursor_txt): - r = parsing.Parser(cursor_txt, no_docstr=True) + offset = self.pos[0] - 1 + r = parsing.Parser(cursor_txt, no_docstr=True, line_offset=offset) try: stmt = r.module.statements[0] except IndexError: raise NotFoundError() - stmt.start_pos = self.pos stmt.parent = self._parser.user_scope return stmt diff --git a/jedi/evaluate.py b/jedi/evaluate.py index 7a6bc069..e915d00f 100644 --- a/jedi/evaluate.py +++ b/jedi/evaluate.py @@ -373,7 +373,8 @@ def find_name(scope, name_str, position=None, search_global=False, if not result and isinstance(nscope, er.Instance): # __getattr__ / __getattribute__ result += check_getattr(nscope, name_str) - debug.dbg('sfn filter "%s" in %s: %s' % (name_str, nscope, result)) + debug.dbg('sfn filter "%s" in %s: %s@%s' % (name_str, nscope, result, + position)) return result def descriptor_check(result): diff --git a/jedi/parsing_representation.py b/jedi/parsing_representation.py index f9a0bb66..cc229c3d 100644 --- a/jedi/parsing_representation.py +++ b/jedi/parsing_representation.py @@ -760,8 +760,7 @@ class Statement(Simple): maybe_dict = array_type == Array.SET break_tok = '' while True: - stmt, break_tok = parse_array_el(token_iterator, - start_pos, maybe_dict) + stmt, break_tok = parse_array_el(token_iterator, maybe_dict) if stmt is None: break else: @@ -782,17 +781,24 @@ class Statement(Simple): else 0) return arr - def parse_array_el(token_iterator, start_pos, maybe_dict=False): + def parse_array_el(token_iterator, maybe_dict=False): token_list = [] level = 1 tok = None + first = True for i, tok_temp in token_iterator: try: token_type, tok, start_tok_pos = tok_temp - end_pos = start_pos[0], start_pos[1] + len(tok) + end_pos = start_tok_pos[0], start_tok_pos[1] + len(tok) + if first: + first = False + start_pos = start_tok_pos except TypeError: # the token is a Name, which has already been parsed tok = tok_temp + if first: + start_pos = tok.start_pos + first = False end_pos = tok.end_pos else: if tok in closing_brackets: @@ -870,7 +876,7 @@ class Statement(Simple): is_chain = True elif tok == ',': # implies a tuple # rewrite `result`, because now the whole thing is a tuple - add_el, t = parse_array_el(enumerate(result), start_pos) + add_el, t = parse_array_el(enumerate(result)) arr = parse_array(token_iterator, Array.TUPLE, start_pos, add_el) result = [arr] diff --git a/jedi/recursion.py b/jedi/recursion.py index 8bc31f1b..70c9aaa4 100644 --- a/jedi/recursion.py +++ b/jedi/recursion.py @@ -35,7 +35,7 @@ class RecursionDecorator(object): def push_stmt(self, stmt): self.current = RecursionNode(stmt, self.current) if self._check_recursion(): - debug.warning('catched recursion', stmt) + debug.warning('catched recursion', stmt, stmt.start_pos) self.pop_stmt() return True return False From 942c620cde0bc4dd2f9761c05a01d9aa28fefe6a Mon Sep 17 00:00:00 2001 From: David Halter Date: Fri, 8 Feb 2013 15:24:49 +0100 Subject: [PATCH 15/83] make the second test pass --- jedi/dynamic.py | 4 ++-- jedi/evaluate.py | 15 +++++++-------- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/jedi/dynamic.py b/jedi/dynamic.py index 7bc507c8..8e9f85dd 100644 --- a/jedi/dynamic.py +++ b/jedi/dynamic.py @@ -311,11 +311,11 @@ def _check_array_additions(compare_array, module, is_list): def check_array_instances(instance): - """ Used for set() and list() instances. """ + """Used for set() and list() instances.""" if not settings.dynamic_arrays_instances: return instance.var_args ai = ArrayInstance(instance) - return helpers.generate_param_array([ai], instance.var_args.parent_stmt) + return [ai] class ArrayInstance(pr.Base): diff --git a/jedi/evaluate.py b/jedi/evaluate.py index e915d00f..5517b812 100644 --- a/jedi/evaluate.py +++ b/jedi/evaluate.py @@ -285,17 +285,16 @@ def find_name(scope, name_str, position=None, search_global=False, inst.is_generated = True result.append(inst) elif par.isinstance(pr.Statement): - def is_execution(arr): - for a in arr: - a = a[0] # rest is always empty with assignees - if a.isinstance(pr.Array): - if is_execution(a): + def is_execution(calls): + for c in calls: + if c.isinstance(pr.Array): + if is_execution(c): return True - elif a.isinstance(pr.Call): + elif c.isinstance(pr.Call): # Compare start_pos, because names may be different # because of executions. - if a.name.start_pos == name.start_pos \ - and a.execution: + if c.name.start_pos == name.start_pos \ + and c.execution: return True return False From d6257fffc8289c8d026e673772a7e977fc18c4b9 Mon Sep 17 00:00:00 2001 From: David Halter Date: Fri, 8 Feb 2013 15:50:35 +0100 Subject: [PATCH 16/83] make array slicing work again --- jedi/evaluate_representation.py | 10 +++++----- jedi/parsing_representation.py | 7 +++++++ 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/jedi/evaluate_representation.py b/jedi/evaluate_representation.py index 172b91e6..c9ec3284 100644 --- a/jedi/evaluate_representation.py +++ b/jedi/evaluate_representation.py @@ -771,14 +771,14 @@ class Array(use_metaclass(cache.CachedMetaClass, pr.Base)): def __init__(self, array): self._array = array - def get_index_types(self, index_call_list=None): + def get_index_types(self, index_arr=None): """ Get the types of a specific index or all, if not given """ - # array slicing - if index_call_list is not None: - if index_call_list and [x for x in index_call_list if ':' in x.token_list]: + if index_arr is not None: + if index_arr and [x for x in index_arr if ':' in x.get_assignment_calls()]: + # array slicing return [self] - index_possibilities = self._follow_values(index_call_list) + index_possibilities = self._follow_values(index_arr) if len(index_possibilities) == 1: # This is indexing only one element, with a fixed index number, # otherwise it just ignores the index (e.g. [1+1]). diff --git a/jedi/parsing_representation.py b/jedi/parsing_representation.py index cc229c3d..81012f2f 100644 --- a/jedi/parsing_representation.py +++ b/jedi/parsing_representation.py @@ -730,6 +730,13 @@ class Statement(Simple): # first keyword of the first token is global -> must be a global return str(self.token_list[0]) == "global" + def get_command(self, index): + commands = self.get_assignment_calls() + try: + return commands[index] + except IndexError: + return None + @property def assignment_details(self): if self._assignment_calls is None: From 70f07320aa4e5387aa8311260be5c324549c8c01 Mon Sep 17 00:00:00 2001 From: David Halter Date: Fri, 8 Feb 2013 16:19:05 +0100 Subject: [PATCH 17/83] Statement.get_assignment_details -> get_commands (makes more sense) --- jedi/api.py | 4 ++-- jedi/dynamic.py | 18 +++++++++--------- jedi/evaluate.py | 15 ++++++++------- jedi/evaluate_representation.py | 12 ++++++------ jedi/modules.py | 7 +++---- jedi/parsing_representation.py | 18 +++++++++--------- jedi/refactoring.py | 13 ++++++------- 7 files changed, 43 insertions(+), 44 deletions(-) diff --git a/jedi/api.py b/jedi/api.py index 44718749..e5f0d457 100644 --- a/jedi/api.py +++ b/jedi/api.py @@ -355,9 +355,9 @@ class Script(object): if user_stmt is None \ or not isinstance(user_stmt, pr.Statement): return None, 0 - ass = helpers.fast_parent_copy(user_stmt.get_assignment_calls()) + commands = helpers.fast_parent_copy(user_stmt.get_commands()) - call, index, stop = helpers.search_function_call(ass, self.pos) + call, index, stop = helpers.search_function_call(commands, self.pos) return call, index def check_cache(): diff --git a/jedi/dynamic.py b/jedi/dynamic.py index 8e9f85dd..cb8e8fcf 100644 --- a/jedi/dynamic.py +++ b/jedi/dynamic.py @@ -133,7 +133,7 @@ def search_params(param): for stmt in possible_stmts: if not isinstance(stmt, pr.Import): - calls = _scan_array(stmt.get_assignment_calls(), func_name) + calls = _scan_array(stmt.get_commands(), func_name) for c in calls: # no execution means that params cannot be set call_path = c.generate_call_path() @@ -157,11 +157,11 @@ def search_params(param): # get the param name if param.assignment_details: - arr = param.assignment_details[0][1] + commands = param.assignment_details[0] else: - arr = param.get_assignment_calls() - offset = 1 if arr[0][0] in ['*', '**'] else 0 - param_name = str(arr[0][offset].name) + commands = param.get_commands() + offset = 1 if commands[0] in ['*', '**'] else 0 + param_name = str(commands[0][offset].name) # add the listener listener = ParamListener() @@ -303,7 +303,7 @@ def _check_array_additions(compare_array, module, is_list): if evaluate.follow_statement.push_stmt(stmt): # check recursion continue - res += check_calls(_scan_array(stmt.get_assignment_calls(), n), n) + res += check_calls(_scan_array(stmt.get_commands(), n), n) evaluate.follow_statement.pop_stmt() # reset settings settings.dynamic_params_for_other_modules = temp_param_add @@ -416,7 +416,7 @@ def related_names(definitions, search_name, mods): if set(f) & set(definitions): names.append(api_classes.RelatedName(name_part, stmt)) else: - calls = _scan_array(stmt.get_assignment_calls(), search_name) + calls = _scan_array(stmt.get_commands(), search_name) for d in stmt.assignment_details: calls += _scan_array(d[1], search_name) for call in calls: @@ -462,9 +462,9 @@ def check_flow_information(flow, search_name, pos): def check_statement_information(stmt, search_name): try: - ass = stmt.get_assignment_calls() + commands = stmt.get_commands() try: - call = ass.get_only_subelement() + call = commands.get_only_subelement() except AttributeError: assert False assert type(call) == pr.Call and str(call.name) == 'isinstance' diff --git a/jedi/evaluate.py b/jedi/evaluate.py index 5517b812..a5370cbf 100644 --- a/jedi/evaluate.py +++ b/jedi/evaluate.py @@ -252,8 +252,8 @@ def find_name(scope, name_str, position=None, search_global=False, return [] result = get_iterator_types(follow_statement(loop.inits[0])) if len(loop.set_vars) > 1: - var_arr = loop.set_stmt.get_assignment_calls() - result = assign_tuples(var_arr, result, name_str) + commands = loop.set_stmt.get_commands() + result = assign_tuples(commands, result, name_str) return result def process(name): @@ -534,11 +534,11 @@ def follow_statement(stmt, seek_name=None): :param seek_name: A string. """ debug.dbg('follow_stmt %s (%s)' % (stmt, seek_name)) - call_list = stmt.get_assignment_calls() - debug.dbg('calls: %s' % call_list) + commands = stmt.get_commands() + debug.dbg('calls: %s' % commands) try: - result = follow_call_list(call_list) + result = follow_call_list(commands) except AttributeError: # This is so evil! But necessary to propagate errors. The attribute # errors here must not be catched, because they shouldn't exist. @@ -741,8 +741,9 @@ def filter_private_variable(scope, call_scope, var_name): def goto(stmt, call_path=None): if call_path is None: - arr = stmt.get_assignment_calls() - call = arr.get_only_subelement() + commands = stmt.get_commands() + assert len(commands) == 1 + call = commands[0] call_path = list(call.generate_call_path()) scope = stmt.parent diff --git a/jedi/evaluate_representation.py b/jedi/evaluate_representation.py index c9ec3284..6541be72 100644 --- a/jedi/evaluate_representation.py +++ b/jedi/evaluate_representation.py @@ -215,9 +215,9 @@ class InstanceElement(use_metaclass(cache.CachedMetaClass)): return self return func - def get_assignment_calls(self): + def get_commands(self): # Copy and modify the array. - origin = self.var.get_assignment_calls() + origin = self.var.get_commands() # Delete parent, because it isn't used anymore. new = helpers.fast_parent_copy(origin) par = InstanceElement(self.instance, origin.parent_stmt, @@ -548,8 +548,8 @@ class Execution(Executable): values=[value])) key, value = next(var_arg_iterator, (None, None)) - assignments = param.get_assignment_calls().values - assignment = assignments[0] + commands = param.get_commands().values + assignment = commands[0] keys = [] values = [] array_type = None @@ -576,7 +576,7 @@ class Execution(Executable): else: if param.assignment_details: # No value: return the default values. - values = assignments + values = commands else: # If there is no assignment detail, that means there is # no assignment, just the result. Therefore nothing has @@ -774,7 +774,7 @@ class Array(use_metaclass(cache.CachedMetaClass, pr.Base)): def get_index_types(self, index_arr=None): """ Get the types of a specific index or all, if not given """ if index_arr is not None: - if index_arr and [x for x in index_arr if ':' in x.get_assignment_calls()]: + if index_arr and [x for x in index_arr if ':' in x.get_commands()]: # array slicing return [self] diff --git a/jedi/modules.py b/jedi/modules.py index e9c8a46b..b40d3f30 100644 --- a/jedi/modules.py +++ b/jedi/modules.py @@ -311,10 +311,9 @@ def sys_path_with_modifications(module): sys_path = list(get_sys_path()) # copy for p in possible_stmts: - try: - call = p.get_assignment_calls().get_only_subelement() - except AttributeError: - continue + commands = p.get_commands() + assert len(commands) == 1 + call = commands[0] n = call.name if not isinstance(n, pr.Name) or len(n.names) != 3: continue diff --git a/jedi/parsing_representation.py b/jedi/parsing_representation.py index 81012f2f..e8102ed8 100644 --- a/jedi/parsing_representation.py +++ b/jedi/parsing_representation.py @@ -658,7 +658,7 @@ class Statement(Simple): :type start_pos: tuple(int, int) """ __slots__ = ('used_funcs', 'code', 'token_list', 'used_vars', - 'set_vars', '_assignment_calls', '_assignment_details') + 'set_vars', '_commands', '_assignment_details') def __init__(self, module, code, set_vars, used_funcs, used_vars, token_list, start_pos, end_pos, parent=None): @@ -674,7 +674,7 @@ class Statement(Simple): self.parent = parent # cache - self._assignment_calls = None + self._commands = None self._assignment_details = None # this is important for other scripts @@ -711,7 +711,7 @@ class Statement(Simple): def get_code(self, new_line=True): code = '' - for c in self.get_assignment_calls(): + for c in self.get_commands(): if isinstance(c, Call): code += c.get_code() else: @@ -731,7 +731,7 @@ class Statement(Simple): return str(self.token_list[0]) == "global" def get_command(self, index): - commands = self.get_assignment_calls() + commands = self.get_commands() try: return commands[index] except IndexError: @@ -739,16 +739,16 @@ class Statement(Simple): @property def assignment_details(self): - if self._assignment_calls is None: + if self._commands is None: # parse statement and therefore get the assignment details. self._parse_statement() return self._assignment_details - def get_assignment_calls(self): - if self._assignment_calls is None: + def get_commands(self): + if self._commands is None: result = self._parse_statement() - self._assignment_calls = result - return self._assignment_calls + self._commands = result + return self._commands def _parse_statement(self): """ diff --git a/jedi/refactoring.py b/jedi/refactoring.py index d88e3f78..0ac687be 100644 --- a/jedi/refactoring.py +++ b/jedi/refactoring.py @@ -113,8 +113,7 @@ def extract(script, new_name): if user_stmt: pos = script.pos line_index = pos[0] - 1 - arr, index = helpers.array_for_pos(user_stmt.get_assignment_calls(), - pos) + arr, index = helpers.array_for_pos(user_stmt.get_commands(), pos) if arr: s = arr.start_pos[0], arr.start_pos[1] + 1 positions = [s] + arr.arr_el_pos + [arr.end_pos] @@ -178,16 +177,16 @@ def inline(script): if not stmt.start_pos <= r.start_pos <= stmt.end_pos] inlines = sorted(inlines, key=lambda x: (x.module_path, x.start_pos), reverse=True) - ass = stmt.get_assignment_calls() + commands = stmt.get_commands() # don't allow multiline refactorings for now. - assert ass.start_pos[0] == ass.end_pos[0] - index = ass.start_pos[0] - 1 + assert commands.start_pos[0] == commands.end_pos[0] + index = commands.start_pos[0] - 1 line = new_lines[index] - replace_str = line[ass.start_pos[1]:ass.end_pos[1] + 1] + replace_str = line[commands.start_pos[1]:commands.end_pos[1] + 1] replace_str = replace_str.strip() # tuples need parentheses - if len(ass.values) > 1: + if len(commands.values) > 1: replace_str = '(%s)' % replace_str # if it's the only assignment, remove the statement From bd393883b6b70f11c77c91e4f30c467288bbab4e Mon Sep 17 00:00:00 2001 From: David Halter Date: Fri, 8 Feb 2013 17:58:27 +0100 Subject: [PATCH 18/83] fixed assignment_detail parsing if tuples were used before = --- jedi/evaluate.py | 1 + jedi/parsing_representation.py | 56 +++++++++++++++++++++------------- 2 files changed, 36 insertions(+), 21 deletions(-) diff --git a/jedi/evaluate.py b/jedi/evaluate.py index a5370cbf..0d587d72 100644 --- a/jedi/evaluate.py +++ b/jedi/evaluate.py @@ -546,6 +546,7 @@ def follow_statement(stmt, seek_name=None): # Assignment checking is only important if the statement defines multiple # variables. + print(seek_name, stmt, stmt.assignment_details) if len(stmt.get_set_vars()) > 1 and seek_name and stmt.assignment_details: new_result = [] for op, set_vars in stmt.assignment_details: diff --git a/jedi/parsing_representation.py b/jedi/parsing_representation.py index e8102ed8..99fb4d4f 100644 --- a/jedi/parsing_representation.py +++ b/jedi/parsing_representation.py @@ -675,7 +675,7 @@ class Statement(Simple): # cache self._commands = None - self._assignment_details = None + self._assignment_details = [] # this is important for other scripts def _remove_executions_from_set_vars(self, set_vars): @@ -710,12 +710,15 @@ class Statement(Simple): return list(result) def get_code(self, new_line=True): - code = '' - for c in self.get_commands(): - if isinstance(c, Call): - code += c.get_code() - else: - code += c + def assemble(assignment, command_list): + pieces = [c.get_code() if isinstance(c, Call) else c + for c in command_list] + if assignment is None: + return ''.join(pieces) + return '%s %s ' % (''.join(pieces), assignment) + + code = ''.join(assemble(*a) for a in self._assignment_details) + code += assemble(None, self.get_commands()) if new_line: return code + '\n' @@ -739,13 +742,13 @@ class Statement(Simple): @property def assignment_details(self): - if self._commands is None: - # parse statement and therefore get the assignment details. - self._parse_statement() + # parse statement which creates the assignment details. + self.get_commands() return self._assignment_details def get_commands(self): if self._commands is None: + self._commands = ['time neeeeed'] # avoid recursions result = self._parse_statement() self._commands = result return self._commands @@ -759,21 +762,25 @@ class Statement(Simple): This is not really nice written, sorry for that. If you plan to replace it and make it nicer, that would be cool :-) """ + def is_assignment(tok): + return tok.endswith('=') and not tok in ['>=', '<=', '==', '!='] + def parse_array(token_iterator, array_type, start_pos, add_el=None): arr = Array(self.module, start_pos, array_type) if add_el is not None: arr.add_statement(add_el) maybe_dict = array_type == Array.SET - break_tok = '' + break_tok = None while True: - stmt, break_tok = parse_array_el(token_iterator, maybe_dict) + stmt, break_tok = parse_array_el(token_iterator, maybe_dict, + break_on_assignment=bool(add_el)) if stmt is None: break else: is_key = maybe_dict and break_tok == ':' arr.add_statement(stmt, is_key) - if break_tok in closing_brackets: + if break_tok in closing_brackets or is_assignment(break_tok): break if not arr.values and maybe_dict: # this is a really special case - empty brackets {} are @@ -786,9 +793,10 @@ class Statement(Simple): else start_pos[0], start_pos[1] + 1 arr.end_pos = end_pos[0], end_pos[1] + (len(break_tok) if break_tok else 0) - return arr + return arr, break_tok - def parse_array_el(token_iterator, maybe_dict=False): + def parse_array_el(token_iterator, maybe_dict=False, + break_on_assignment=False): token_list = [] level = 1 tok = None @@ -813,7 +821,9 @@ class Statement(Simple): elif tok in brackets.keys(): level += 1 - if level == 0 and tok in closing_brackets or level == 1 and tok == ',': + if level == 0 and tok in closing_brackets \ + or level == 1 and (tok == ',' or is_assignment(tok) + and break_on_assignment): break token_list.append(tok_temp) @@ -826,7 +836,6 @@ class Statement(Simple): return statement, tok # initializations - self._assignment_details = [] result = [] is_chain = False brackets = {'(': Array.TUPLE, '[': Array.LIST, '{': Array.SET} @@ -843,7 +852,7 @@ class Statement(Simple): token_type = None start_pos = tok.start_pos else: - if tok.endswith('=') and not tok in ['>=', '<=', '==', '!=']: + if is_assignment(tok): # This means, there is an assignment here. # Add assignments, which can be more than one self._assignment_details.append((tok, result)) @@ -871,7 +880,8 @@ class Statement(Simple): result.append(call) is_chain = False elif tok in brackets.keys(): - arr = parse_array(token_iterator, brackets[tok], start_pos) + arr, is_ass = parse_array(token_iterator, brackets[tok], + start_pos) if result and isinstance(result[-1], Call): result[-1].set_execution(arr) else: @@ -884,9 +894,13 @@ class Statement(Simple): elif tok == ',': # implies a tuple # rewrite `result`, because now the whole thing is a tuple add_el, t = parse_array_el(enumerate(result)) - arr = parse_array(token_iterator, Array.TUPLE, start_pos, - add_el) + arr, break_tok = parse_array(token_iterator, Array.TUPLE, + start_pos, add_el) result = [arr] + if is_assignment(break_tok): + self._assignment_details.append((break_tok, result)) + result = [] + is_chain = False else: if tok != '\n': result.append(tok) From c831bfcca1743ca6fe058af0e7bf314f6cc6c419 Mon Sep 17 00:00:00 2001 From: David Halter Date: Fri, 8 Feb 2013 18:00:27 +0100 Subject: [PATCH 19/83] exchanged assignment_details order (another refactoring) --- jedi/evaluate.py | 6 +++--- jedi/parsing_representation.py | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/jedi/evaluate.py b/jedi/evaluate.py index 0d587d72..b178e15a 100644 --- a/jedi/evaluate.py +++ b/jedi/evaluate.py @@ -299,7 +299,7 @@ def find_name(scope, name_str, position=None, search_global=False, return False is_exe = False - for op, assignee in par.assignment_details: + for assignee, op in par.assignment_details: is_exe |= is_execution(assignee) if is_exe: @@ -308,7 +308,7 @@ def find_name(scope, name_str, position=None, search_global=False, pass else: details = par.assignment_details - if details and details[0][0] != '=': + if details and details[0][1] != '=': no_break_scope = True # TODO this makes self variables non-breakable. wanted? @@ -549,7 +549,7 @@ def follow_statement(stmt, seek_name=None): print(seek_name, stmt, stmt.assignment_details) if len(stmt.get_set_vars()) > 1 and seek_name and stmt.assignment_details: new_result = [] - for op, set_vars in stmt.assignment_details: + for set_vars, op in stmt.assignment_details: new_result += assign_tuples(set_vars, result, seek_name) result = new_result return set(result) diff --git a/jedi/parsing_representation.py b/jedi/parsing_representation.py index 99fb4d4f..d8b4fe16 100644 --- a/jedi/parsing_representation.py +++ b/jedi/parsing_representation.py @@ -710,7 +710,7 @@ class Statement(Simple): return list(result) def get_code(self, new_line=True): - def assemble(assignment, command_list): + def assemble(command_list, assignment=None): pieces = [c.get_code() if isinstance(c, Call) else c for c in command_list] if assignment is None: @@ -718,7 +718,7 @@ class Statement(Simple): return '%s %s ' % (''.join(pieces), assignment) code = ''.join(assemble(*a) for a in self._assignment_details) - code += assemble(None, self.get_commands()) + code += assemble(self.get_commands()) if new_line: return code + '\n' @@ -855,7 +855,7 @@ class Statement(Simple): if is_assignment(tok): # This means, there is an assignment here. # Add assignments, which can be more than one - self._assignment_details.append((tok, result)) + self._assignment_details.append((result, tok)) result = [] is_chain = False continue @@ -898,7 +898,7 @@ class Statement(Simple): start_pos, add_el) result = [arr] if is_assignment(break_tok): - self._assignment_details.append((break_tok, result)) + self._assignment_details.append((result, break_tok)) result = [] is_chain = False else: From a5e9977e94b8a3102d703f1f4e1dfde593e383ec Mon Sep 17 00:00:00 2001 From: David Halter Date: Fri, 8 Feb 2013 18:46:56 +0100 Subject: [PATCH 20/83] fix first tuple assignments --- jedi/dynamic.py | 2 +- jedi/evaluate.py | 59 ++++++++++++++++----------------- jedi/evaluate_representation.py | 4 +-- 3 files changed, 31 insertions(+), 34 deletions(-) diff --git a/jedi/dynamic.py b/jedi/dynamic.py index cb8e8fcf..aecd18f6 100644 --- a/jedi/dynamic.py +++ b/jedi/dynamic.py @@ -418,7 +418,7 @@ def related_names(definitions, search_name, mods): else: calls = _scan_array(stmt.get_commands(), search_name) for d in stmt.assignment_details: - calls += _scan_array(d[1], search_name) + calls += _scan_array(d[0], search_name) for call in calls: names += check_call(call) return names diff --git a/jedi/evaluate.py b/jedi/evaluate.py index b178e15a..f7f4b218 100644 --- a/jedi/evaluate.py +++ b/jedi/evaluate.py @@ -483,41 +483,38 @@ def assign_tuples(tup, results, seek_name): def eval_results(index): types = [] for r in results: - if hasattr(r, "get_exact_index_types"): - try: - types += r.get_exact_index_types(index) - except IndexError: - pass - else: + try: + func = r.get_exact_index_types + except AttributeError: debug.warning("invalid tuple lookup %s of result %s in %s" % (tup, results, seek_name)) - + else: + try: + types += func(index) + except IndexError: + pass return types result = [] - if tup.type == pr.Array.NOARRAY: - # Here we have unnessecary braces, which we just remove. - arr = tup.get_only_subelement() - if type(arr) == pr.Call: - if arr.name.names[-1] == seek_name: - result = results - else: - result = assign_tuples(arr, results, seek_name) - else: - for i, t in enumerate(tup): - # Used in assignments. There is just one call and no other things, - # therefore we can just assume, that the first part is important. - if len(t) != 1: - raise AttributeError('Array length should be 1') - t = t[0] + for i, stmt in enumerate(tup): + # Used in assignments. There is just one call and no other things, + # therefore we can just assume, that the first part is important. + command = stmt.get_commands()[0] - # Check the left part, if there are still tuples in it or a Call. - if isinstance(t, pr.Array): - # These are "sub"-tuples. - result += assign_tuples(t, eval_results(i), seek_name) - else: - if t.name.names[-1] == seek_name: - result += eval_results(i) + if tup.type == pr.Array.NOARRAY: + + # unnessecary braces -> just remove. + r = results + else: + r = eval_results(i) + + # are there still tuples or is it just a Call. + if isinstance(command, pr.Array): + # These are "sub"-tuples. + result += assign_tuples(command, r, seek_name) + else: + if command.name.names[-1] == seek_name: + result += r return result @@ -549,8 +546,8 @@ def follow_statement(stmt, seek_name=None): print(seek_name, stmt, stmt.assignment_details) if len(stmt.get_set_vars()) > 1 and seek_name and stmt.assignment_details: new_result = [] - for set_vars, op in stmt.assignment_details: - new_result += assign_tuples(set_vars, result, seek_name) + for ass_commands, op in stmt.assignment_details: + new_result += assign_tuples(ass_commands[0], result, seek_name) result = new_result return set(result) diff --git a/jedi/evaluate_representation.py b/jedi/evaluate_representation.py index 6541be72..1d627b53 100644 --- a/jedi/evaluate_representation.py +++ b/jedi/evaluate_representation.py @@ -630,10 +630,10 @@ class Execution(Executable): # Normal arguments (including key arguments). else: if stmt.assignment_detail: - tok, key_arr = stmt.assignment_detail[0] + key_arr, op = stmt.assignment_detail[0] # named parameter if key_arr and isinstance(key_arr[0], pr.Call): - yield tok[0].name, stmt + yield op[0].name, stmt else: yield None, stmt From 2fef25b1b1f76c6a7f2ecaadf826f0d8400a6245 Mon Sep 17 00:00:00 2001 From: David Halter Date: Fri, 8 Feb 2013 21:18:58 +0100 Subject: [PATCH 21/83] fixed some dict issues --- jedi/evaluate.py | 1 - jedi/evaluate_representation.py | 13 +++++++------ jedi/parsing_representation.py | 20 +++++++++++--------- 3 files changed, 18 insertions(+), 16 deletions(-) diff --git a/jedi/evaluate.py b/jedi/evaluate.py index f7f4b218..59e2d8f1 100644 --- a/jedi/evaluate.py +++ b/jedi/evaluate.py @@ -543,7 +543,6 @@ def follow_statement(stmt, seek_name=None): # Assignment checking is only important if the statement defines multiple # variables. - print(seek_name, stmt, stmt.assignment_details) if len(stmt.get_set_vars()) > 1 and seek_name and stmt.assignment_details: new_result = [] for ass_commands, op in stmt.assignment_details: diff --git a/jedi/evaluate_representation.py b/jedi/evaluate_representation.py index 1d627b53..d825e09a 100644 --- a/jedi/evaluate_representation.py +++ b/jedi/evaluate_representation.py @@ -795,13 +795,14 @@ class Array(use_metaclass(cache.CachedMetaClass, pr.Base)): result += dynamic.check_array_additions(self) return set(result) - def get_exact_index_types(self, index): + def get_exact_index_types(self, mixed_index): """ Here the index is an int. Raises IndexError/KeyError """ - if self._array.type == pr.Array.DICT: - old_index = index + index = mixed_index + if self.type == pr.Array.DICT: index = None - for i, key_elements in enumerate(self._array.keys): + for i, key_statement in enumerate(self._array.keys): # Because we only want the key to be a string. + key_elements = key_statement.get_commands() if len(key_elements) == 1: try: str_key = key_elements.get_code() @@ -810,12 +811,12 @@ class Array(use_metaclass(cache.CachedMetaClass, pr.Base)): str_key = key_elements[0].name except AttributeError: str_key = None - if old_index == str_key: + if mixed_index == str_key: index = i break if index is None: raise KeyError('No key found in dictionary') - values = [self._array[index]] + values = [self._array.values[index]] return self._follow_values(values) def _follow_values(self, values): diff --git a/jedi/parsing_representation.py b/jedi/parsing_representation.py index d8b4fe16..1308d13a 100644 --- a/jedi/parsing_representation.py +++ b/jedi/parsing_representation.py @@ -822,8 +822,9 @@ class Statement(Simple): level += 1 if level == 0 and tok in closing_brackets \ - or level == 1 and (tok == ',' or is_assignment(tok) - and break_on_assignment): + or level == 1 and (tok == ',' + or maybe_dict and tok == ':' + or is_assignment(tok) and break_on_assignment): break token_list.append(tok_temp) @@ -1038,6 +1039,7 @@ class Array(Call): """Just add a new statement""" statement.parent = self if is_key: + self.type = self.DICT self.keys.append(statement) else: self.values.append(statement) @@ -1059,20 +1061,20 @@ class Array(Call): def __getitem__(self, key): if self.type == self.DICT: - raise NotImplementedError('no dicts allowed, yet') + raise TypeError('no dicts allowed') return self.values[key] def __iter__(self): if self.type == self.DICT: - raise NotImplementedError('no dicts allowed, yet') + raise TypeError('no dicts allowed') return iter(self.values) def get_code(self): - map = {Array.NOARRAY: '%s', - Array.TUPLE: '(%s)', - Array.LIST: '[%s]', - Array.DICT: '{%s}', - Array.SET: '{%s}' + map = {self.NOARRAY: '%s', + self.TUPLE: '(%s)', + self.LIST: '[%s]', + self.DICT: '{%s}', + self.SET: '{%s}' } inner = [] for i, stmt in enumerate(self.values): From 07f4c080691663a7cc0e1efb22931376afd19954 Mon Sep 17 00:00:00 2001 From: David Halter Date: Fri, 8 Feb 2013 23:16:04 +0100 Subject: [PATCH 22/83] some param fixes --- jedi/evaluate_representation.py | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/jedi/evaluate_representation.py b/jedi/evaluate_representation.py index d825e09a..d8833cba 100644 --- a/jedi/evaluate_representation.py +++ b/jedi/evaluate_representation.py @@ -495,16 +495,20 @@ class Execution(Executable): """ Create a param with the original scope (of varargs) as parent. """ - parent_stmt = self.var_args.parent_stmt - pos = parent_stmt.start_pos if parent_stmt else None - calls = pr.Array(pos, pr.Array.NOARRAY, parent_stmt) + if isinstance(self.var_args, pr.Array): + parent = self.var_args.parent + start_pos = self.var_args.start_pos + else: + parent = self.base + start_pos = None + calls = pr.Array(start_pos, pr.Array.NOARRAY, parent) calls.values = values calls.keys = keys calls.type = array_type new_param = copy.copy(param) - if parent_stmt is not None: - new_param.parent = parent_stmt - new_param._assignment_calls = calls + if parent is not None: + new_param.parent = parent + new_param._commands = calls new_param.is_generated = True name = copy.copy(param.get_name()) name.parent = new_param @@ -548,12 +552,11 @@ class Execution(Executable): values=[value])) key, value = next(var_arg_iterator, (None, None)) - commands = param.get_commands().values - assignment = commands[0] + commands = param.get_commands() keys = [] values = [] array_type = None - if assignment[0] == '*': + if commands[0] == '*': # *args param array_type = pr.Array.TUPLE if value: @@ -564,7 +567,7 @@ class Execution(Executable): var_arg_iterator.push_back((key, value)) break values.append(value) - elif assignment[0] == '**': + elif commands[0] == '**': # **kwargs param array_type = pr.Array.DICT if non_matching_keys: @@ -585,7 +588,7 @@ class Execution(Executable): # Just ignore all the params that are without a key, after one # keyword argument was set. - if not keys_only or assignment[0] == '**': + if not keys_only or commands[0] == '**': keys_used.add(str(key)) result.append(gen_param_name_copy(param, keys=keys, values=values, array_type=array_type)) From ffaaa68f56edbae4ae78e840ebd5d61f603fd6d6 Mon Sep 17 00:00:00 2001 From: David Halter Date: Sat, 9 Feb 2013 00:00:26 +0100 Subject: [PATCH 23/83] fix unnessecary bracket stuff --- jedi/evaluate.py | 4 +++- jedi/parsing_representation.py | 19 +++++++++++-------- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/jedi/evaluate.py b/jedi/evaluate.py index 59e2d8f1..b1fb43aa 100644 --- a/jedi/evaluate.py +++ b/jedi/evaluate.py @@ -558,6 +558,7 @@ def follow_call_list(call_list, follow_array=False): It is used to evaluate a two dimensional object, that has calls, arrays and operators in it. """ + # TODO remove follow_array?! def evaluate_list_comprehension(lc, parent=None): input = lc.input nested_lc = lc.input.token_list[0] @@ -584,7 +585,8 @@ def follow_call_list(call_list, follow_array=False): calls_iterator = iter(call_list) for call in calls_iterator: if pr.Array.is_type(call, pr.Array.NOARRAY): - result += follow_call_list(call, follow_array=True) + result += itertools.chain.from_iterable(follow_statement(s) + for s in call) elif isinstance(call, pr.ListComprehension): loop = evaluate_list_comprehension(call) stmt = copy.copy(call.stmt) diff --git a/jedi/parsing_representation.py b/jedi/parsing_representation.py index 1308d13a..61995581 100644 --- a/jedi/parsing_representation.py +++ b/jedi/parsing_representation.py @@ -763,7 +763,8 @@ class Statement(Simple): it and make it nicer, that would be cool :-) """ def is_assignment(tok): - return tok.endswith('=') and not tok in ['>=', '<=', '==', '!='] + return tok is not None and tok.endswith('=') \ + and not tok in ['>=', '<=', '==', '!='] def parse_array(token_iterator, array_type, start_pos, add_el=None): arr = Array(self.module, start_pos, array_type) @@ -782,6 +783,8 @@ class Statement(Simple): arr.add_statement(stmt, is_key) if break_tok in closing_brackets or is_assignment(break_tok): break + if arr.type == Array.TUPLE and len(arr) == 1 and break_tok != ',': + arr.type = Array.NOARRAY if not arr.values and maybe_dict: # this is a really special case - empty brackets {} are # always dictionaries and not sets. @@ -802,13 +805,7 @@ class Statement(Simple): tok = None first = True for i, tok_temp in token_iterator: - try: - token_type, tok, start_tok_pos = tok_temp - end_pos = start_tok_pos[0], start_tok_pos[1] + len(tok) - if first: - first = False - start_pos = start_tok_pos - except TypeError: + if isinstance(tok_temp, (Base)): # the token is a Name, which has already been parsed tok = tok_temp if first: @@ -816,6 +813,12 @@ class Statement(Simple): first = False end_pos = tok.end_pos else: + token_type, tok, start_tok_pos = tok_temp + end_pos = start_tok_pos[0], start_tok_pos[1] + len(tok) + if first: + first = False + start_pos = start_tok_pos + if tok in closing_brackets: level -= 1 elif tok in brackets.keys(): From 25b2239f6adeee4295d7bd5f7cfe6b87f85275f2 Mon Sep 17 00:00:00 2001 From: David Halter Date: Sat, 9 Feb 2013 01:54:48 +0100 Subject: [PATCH 24/83] repr corrections for Array/Statement/Call --- jedi/parsing_representation.py | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/jedi/parsing_representation.py b/jedi/parsing_representation.py index 61995581..da6d8e91 100644 --- a/jedi/parsing_representation.py +++ b/jedi/parsing_representation.py @@ -793,7 +793,7 @@ class Statement(Simple): k, v = arr.keys, arr.values latest = (v[-1] if v else k[-1] if k else None) end_pos = latest.end_pos if latest is not None \ - else start_pos[0], start_pos[1] + 1 + else (start_pos[0], start_pos[1] + 1) arr.end_pos = end_pos[0], end_pos[1] + (len(break_tok) if break_tok else 0) return arr, break_tok @@ -848,14 +848,13 @@ class Statement(Simple): token_iterator = enumerate(self.token_list) for i, tok_temp in token_iterator: #print 'tok', tok_temp, result - try: - token_type, tok, start_pos = tok_temp - except TypeError: + if isinstance(tok_temp, Base): # the token is a Name, which has already been parsed tok = tok_temp token_type = None start_pos = tok.start_pos else: + token_type, tok, start_pos = tok_temp if is_assignment(tok): # This means, there is an assignment here. # Add assignments, which can be more than one @@ -1003,9 +1002,9 @@ class Call(Simple): if self.type == Call.NAME: s = self.name.get_code() else: - s = repr(self.name) + s = '' if self.name is None else repr(self.name) if self.execution is not None: - s += '(%s)' % self.execution.get_code() + s += self.execution.get_code() if self.next is not None: s += self.next.get_code() return s @@ -1022,8 +1021,8 @@ class Array(Call): http://docs.python.org/py3k/reference/grammar.html Array saves sub-arrays as well as normal operators and calls to methods. - :param array_type: The type of an array, which can be one of the constants\ - below. + :param array_type: The type of an array, which can be one of the constants + below. :type array_type: int """ NOARRAY = None # just brackets, like `1 * (3 + 2)` @@ -1052,7 +1051,6 @@ class Array(Call): """ This is not only used for calls on the actual object, but for ducktyping, to invoke this function with anything as `self`. - TODO remove? """ if isinstance(instance, Array): if instance.type in types: @@ -1073,7 +1071,7 @@ class Array(Call): return iter(self.values) def get_code(self): - map = {self.NOARRAY: '%s', + map = {self.NOARRAY: '(%s)', self.TUPLE: '(%s)', self.LIST: '[%s]', self.DICT: '{%s}', @@ -1090,7 +1088,8 @@ class Array(Call): s += key.get_code(new_line=False) + ': ' s += stmt.get_code(new_line=False) inner.append(s) - return map[self.type] % ', '.join(inner) + s = map[self.type] % ', '.join(inner) + return s + super(Array, self).get_code() def __repr__(self): if self.type == self.NOARRAY: From 9540025a029d3ba9024472f4f20480d0ff471ecc Mon Sep 17 00:00:00 2001 From: David Halter Date: Sat, 9 Feb 2013 02:17:29 +0100 Subject: [PATCH 25/83] fixed some more array tests --- jedi/evaluate.py | 17 ++++++----------- jedi/parsing_representation.py | 8 ++++++-- 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/jedi/evaluate.py b/jedi/evaluate.py index b1fb43aa..146e5ee9 100644 --- a/jedi/evaluate.py +++ b/jedi/evaluate.py @@ -558,7 +558,6 @@ def follow_call_list(call_list, follow_array=False): It is used to evaluate a two dimensional object, that has calls, arrays and operators in it. """ - # TODO remove follow_array?! def evaluate_list_comprehension(lc, parent=None): input = lc.input nested_lc = lc.input.token_list[0] @@ -585,8 +584,12 @@ def follow_call_list(call_list, follow_array=False): calls_iterator = iter(call_list) for call in calls_iterator: if pr.Array.is_type(call, pr.Array.NOARRAY): - result += itertools.chain.from_iterable(follow_statement(s) - for s in call) + r = list(itertools.chain.from_iterable(follow_statement(s) + for s in call)) + call_path = call.generate_call_path() + next(call_path, None) # the first one has been used already + result += follow_paths(call_path, r, call.parent, + position=call.start_pos) elif isinstance(call, pr.ListComprehension): loop = evaluate_list_comprehension(call) stmt = copy.copy(call.stmt) @@ -623,14 +626,6 @@ def follow_call_list(call_list, follow_array=False): and str(r.name) == 'str']: # if it is an iterable, ignore * operations next(calls_iterator) - - if follow_array and isinstance(call_list, pr.Array): - # call_list can also be a two dimensional array - call_path = call_list.generate_call_path() - next(call_path, None) # the first one has been used already - call_scope = call_list.parent_stmt - position = call_list.start_pos - result = follow_paths(call_path, result, call_scope, position=position) return set(result) diff --git a/jedi/parsing_representation.py b/jedi/parsing_representation.py index da6d8e91..f36bf434 100644 --- a/jedi/parsing_representation.py +++ b/jedi/parsing_representation.py @@ -773,17 +773,20 @@ class Statement(Simple): maybe_dict = array_type == Array.SET break_tok = None + is_array = None while True: stmt, break_tok = parse_array_el(token_iterator, maybe_dict, break_on_assignment=bool(add_el)) if stmt is None: break else: + if break_tok == ',': + is_array = True is_key = maybe_dict and break_tok == ':' arr.add_statement(stmt, is_key) if break_tok in closing_brackets or is_assignment(break_tok): break - if arr.type == Array.TUPLE and len(arr) == 1 and break_tok != ',': + if arr.type == Array.TUPLE and len(arr) == 1 and not is_array: arr.type = Array.NOARRAY if not arr.values and maybe_dict: # this is a really special case - empty brackets {} are @@ -1088,7 +1091,8 @@ class Array(Call): s += key.get_code(new_line=False) + ': ' s += stmt.get_code(new_line=False) inner.append(s) - s = map[self.type] % ', '.join(inner) + add = ',' if self.type == self.TUPLE and len(self) == 1 else '' + s = map[self.type] % (', '.join(inner) + add) return s + super(Array, self).get_code() def __repr__(self): From a1e366f7912aba803890abd1dbe03abb9494f1ec Mon Sep 17 00:00:00 2001 From: David Halter Date: Sat, 9 Feb 2013 03:16:10 +0100 Subject: [PATCH 26/83] different little beauty improvements --- jedi/evaluate.py | 4 ++-- jedi/evaluate_representation.py | 16 ++++------------ 2 files changed, 6 insertions(+), 14 deletions(-) diff --git a/jedi/evaluate.py b/jedi/evaluate.py index 146e5ee9..9f94a5ae 100644 --- a/jedi/evaluate.py +++ b/jedi/evaluate.py @@ -372,8 +372,8 @@ def find_name(scope, name_str, position=None, search_global=False, if not result and isinstance(nscope, er.Instance): # __getattr__ / __getattribute__ result += check_getattr(nscope, name_str) - debug.dbg('sfn filter "%s" in %s: %s@%s' % (name_str, nscope, result, - position)) + debug.dbg('sfn filter "%s" in (%s-%s): %s@%s' % (name_str, nscope, + scope, result, position)) return result def descriptor_check(result): diff --git a/jedi/evaluate_representation.py b/jedi/evaluate_representation.py index d8833cba..510abb82 100644 --- a/jedi/evaluate_representation.py +++ b/jedi/evaluate_representation.py @@ -701,6 +701,7 @@ class Execution(Executable): @property @cache.memoize_default() def returns(self): + print self.copy_properties('returns')[0].parent return self.copy_properties('returns') @property @@ -841,18 +842,9 @@ class Array(use_metaclass(cache.CachedMetaClass, pr.Base)): def get_contents(self): return self._array - @property - def parent(self): - """ - Return the builtin scope as parent, because the arrays are builtins - """ - return builtin.Builtin.scope - - def get_parent_until(self, *args, **kwargs): - return builtin.Builtin.scope - def __getattr__(self, name): - if name not in ['type', 'start_pos', 'get_only_subelement']: + if name not in ['type', 'start_pos', 'get_only_subelement', 'parent', + 'get_parent_until']: raise AttributeError('Strange access on %s: %s.' % (self, name)) return getattr(self._array, name) @@ -872,7 +864,7 @@ class ArrayMethod(object): def __getattr__(self, name): # Set access privileges: if name not in ['parent', 'names', 'start_pos', 'end_pos', 'get_code']: - raise AttributeError('Strange access: %s.' % name) + raise AttributeError('Strange accesson %s: %s.' % (self, name)) return getattr(self.name, name) def get_parent_until(self): From 22a1b2397d647e90c3d56d65787983d792c2c219 Mon Sep 17 00:00:00 2001 From: David Halter Date: Sat, 9 Feb 2013 03:55:30 +0100 Subject: [PATCH 27/83] simple function tests pass now --- jedi/evaluate.py | 6 +++--- jedi/evaluate_representation.py | 23 ++++++++++++----------- jedi/helpers.py | 6 ++---- jedi/parsing_representation.py | 4 ++-- 4 files changed, 19 insertions(+), 20 deletions(-) diff --git a/jedi/evaluate.py b/jedi/evaluate.py index 9f94a5ae..f1090a68 100644 --- a/jedi/evaluate.py +++ b/jedi/evaluate.py @@ -372,8 +372,8 @@ def find_name(scope, name_str, position=None, search_global=False, if not result and isinstance(nscope, er.Instance): # __getattr__ / __getattribute__ result += check_getattr(nscope, name_str) - debug.dbg('sfn filter "%s" in (%s-%s): %s@%s' % (name_str, nscope, - scope, result, position)) + debug.dbg('sfn filter "%s" in (%s-%s): %s@%s' % (name_str, scope, + nscope, result, position)) return result def descriptor_check(result): @@ -632,7 +632,7 @@ def follow_call_list(call_list, follow_array=False): def follow_call(call): """Follow a call is following a function, variable, string, etc.""" path = call.generate_call_path() - scope = call.get_parent_until(pr.Scope) + scope = call.get_parent_until((pr.Scope, er.Execution)) return follow_call_path(path, scope, call.start_pos) diff --git a/jedi/evaluate_representation.py b/jedi/evaluate_representation.py index 510abb82..aff57a49 100644 --- a/jedi/evaluate_representation.py +++ b/jedi/evaluate_representation.py @@ -501,14 +501,16 @@ class Execution(Executable): else: parent = self.base start_pos = None - calls = pr.Array(start_pos, pr.Array.NOARRAY, parent) - calls.values = values - calls.keys = keys - calls.type = array_type + # create an Array (-> container for the statement) + arr = pr.Array(self.module, start_pos, pr.Array.NOARRAY, parent) + arr.values = values + arr.keys = keys + arr.type = array_type + new_param = copy.copy(param) if parent is not None: new_param.parent = parent - new_param._commands = calls + new_param._commands = [arr] new_param.is_generated = True name = copy.copy(param.get_name()) name.parent = new_param @@ -610,7 +612,7 @@ class Execution(Executable): if not isinstance(stmt, pr.Statement): yield None, stmt # *args - elif stmt.token_list[0] == '*': + elif stmt.get_commands()[0] == '*': arrays = evaluate.follow_call_list([stmt.token_list[1:]]) # *args must be some sort of an array, otherwise -> ignore for array in arrays: @@ -618,7 +620,7 @@ class Execution(Executable): for field in array.get_contents(): yield None, field # **kwargs - elif stmt[0] == '**': + elif stmt.get_commands()[0] == '**': arrays = evaluate.follow_call_list([stmt.token_list[1:]]) for array in arrays: if hasattr(array, 'get_contents'): @@ -632,8 +634,8 @@ class Execution(Executable): yield name, field # Normal arguments (including key arguments). else: - if stmt.assignment_detail: - key_arr, op = stmt.assignment_detail[0] + if stmt.assignment_details: + key_arr, op = stmt.assignment_details[0] # named parameter if key_arr and isinstance(key_arr[0], pr.Call): yield op[0].name, stmt @@ -677,7 +679,7 @@ class Execution(Executable): raise common.MultiLevelAttributeError(sys.exc_info()) def __getattr__(self, name): - if name not in ['start_pos', 'end_pos', 'imports']: + if name not in ['start_pos', 'end_pos', 'imports', 'module']: raise AttributeError('Tried to access %s: %s. Why?' % (name, self)) return getattr(self.base, name) @@ -701,7 +703,6 @@ class Execution(Executable): @property @cache.memoize_default() def returns(self): - print self.copy_properties('returns')[0].parent return self.copy_properties('returns') @property diff --git a/jedi/helpers.py b/jedi/helpers.py index 4478ba20..3ac02694 100644 --- a/jedi/helpers.py +++ b/jedi/helpers.py @@ -31,8 +31,7 @@ def fast_parent_copy(obj): for key, value in items: # replace parent (first try _parent and then parent) - if key in ['parent', '_parent', '_parent_stmt'] \ - and value is not None: + if key in ['parent', '_parent'] and value is not None: if key == 'parent' and '_parent' in items: # parent can be a property continue @@ -40,8 +39,7 @@ def fast_parent_copy(obj): setattr(new_obj, key, new_elements[value]) except KeyError: pass - elif key in ['parent_stmt', 'parent_function', 'use_as_parent', - 'module']: + elif key in ['parent_function', 'use_as_parent', 'module']: continue elif isinstance(value, list): setattr(new_obj, key, list_rec(value)) diff --git a/jedi/parsing_representation.py b/jedi/parsing_representation.py index f36bf434..50de9347 100644 --- a/jedi/parsing_representation.py +++ b/jedi/parsing_representation.py @@ -711,7 +711,7 @@ class Statement(Simple): def get_code(self, new_line=True): def assemble(command_list, assignment=None): - pieces = [c.get_code() if isinstance(c, Call) else c + pieces = [c.get_code() if isinstance(c, Simple) else c for c in command_list] if assignment is None: return ''.join(pieces) @@ -1036,7 +1036,7 @@ class Array(Call): def __init__(self, module, start_pos, arr_type=NOARRAY, parent=None, values=None): super(Array, self).__init__(module, None, arr_type, start_pos, parent) - self.values = values if values else [] + self.values = values or [] self.keys = [] self.end_pos = None, None From 0008531a3041f27470ad7ddbc2fdbc1174c0de38 Mon Sep 17 00:00:00 2001 From: David Halter Date: Sat, 9 Feb 2013 03:55:56 +0100 Subject: [PATCH 28/83] pr.Array takes no values param anymore --- jedi/parsing_representation.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jedi/parsing_representation.py b/jedi/parsing_representation.py index 50de9347..53e0ac9c 100644 --- a/jedi/parsing_representation.py +++ b/jedi/parsing_representation.py @@ -1034,9 +1034,9 @@ class Array(Call): DICT = 'dict' SET = 'set' - def __init__(self, module, start_pos, arr_type=NOARRAY, parent=None, values=None): + def __init__(self, module, start_pos, arr_type=NOARRAY, parent=None): super(Array, self).__init__(module, None, arr_type, start_pos, parent) - self.values = values or [] + self.values = [] self.keys = [] self.end_pos = None, None From cd4912673393cb791682e93c47d1c1110ced0691 Mon Sep 17 00:00:00 2001 From: David Halter Date: Sun, 10 Feb 2013 02:01:27 +0100 Subject: [PATCH 29/83] improved param generator --- jedi/evaluate_representation.py | 29 ++++++++++++++++++++++------- jedi/parsing_representation.py | 6 +++--- 2 files changed, 25 insertions(+), 10 deletions(-) diff --git a/jedi/evaluate_representation.py b/jedi/evaluate_representation.py index aff57a49..4af13a96 100644 --- a/jedi/evaluate_representation.py +++ b/jedi/evaluate_representation.py @@ -495,23 +495,38 @@ class Execution(Executable): """ Create a param with the original scope (of varargs) as parent. """ + # TODO remove array and param and just put the values of the \ + # statement into the values of the param - it's as simple as that. if isinstance(self.var_args, pr.Array): parent = self.var_args.parent start_pos = self.var_args.start_pos else: parent = self.base start_pos = None - # create an Array (-> container for the statement) - arr = pr.Array(self.module, start_pos, pr.Array.NOARRAY, parent) - arr.values = values - arr.keys = keys - arr.type = array_type new_param = copy.copy(param) + new_param.is_generated = True if parent is not None: new_param.parent = parent - new_param._commands = [arr] - new_param.is_generated = True + + if array_type is not None: + # create an Array (-> needed for *args/**kwargs tuples/dicts) + arr = pr.Array(self.module, start_pos, array_type, parent) + """ + for i, value in enumerate(values): # TODO delete? + try: + keys[i] + except IndexError: + pass + """ + arr.values = values + arr.keys = keys + arr.type = array_type + + new_param._commands = [arr] + else: + new_param._commands = values + name = copy.copy(param.get_name()) name.parent = new_param return name diff --git a/jedi/parsing_representation.py b/jedi/parsing_representation.py index 53e0ac9c..7850d941 100644 --- a/jedi/parsing_representation.py +++ b/jedi/parsing_representation.py @@ -24,7 +24,7 @@ import re import tokenize from _compatibility import next, literal_eval, cleandoc, Python3Method, \ - property + property, unicode import common import debug @@ -711,7 +711,7 @@ class Statement(Simple): def get_code(self, new_line=True): def assemble(command_list, assignment=None): - pieces = [c.get_code() if isinstance(c, Simple) else c + pieces = [c.get_code() if isinstance(c, Base) else c for c in command_list] if assignment is None: return ''.join(pieces) @@ -763,7 +763,7 @@ class Statement(Simple): it and make it nicer, that would be cool :-) """ def is_assignment(tok): - return tok is not None and tok.endswith('=') \ + return isinstance(tok, (str, unicode)) and tok.endswith('=') \ and not tok in ['>=', '<=', '==', '!='] def parse_array(token_iterator, array_type, start_pos, add_el=None): From 7dce3cb964ed45f2b635cafbb33e6abe287b82b4 Mon Sep 17 00:00:00 2001 From: David Halter Date: Sun, 10 Feb 2013 15:50:44 +0100 Subject: [PATCH 30/83] param keys of dicts are statements now --- jedi/evaluate_representation.py | 39 ++++++++++++++++----------------- 1 file changed, 19 insertions(+), 20 deletions(-) diff --git a/jedi/evaluate_representation.py b/jedi/evaluate_representation.py index 4af13a96..3331a39c 100644 --- a/jedi/evaluate_representation.py +++ b/jedi/evaluate_representation.py @@ -509,23 +509,19 @@ class Execution(Executable): if parent is not None: new_param.parent = parent - if array_type is not None: - # create an Array (-> needed for *args/**kwargs tuples/dicts) - arr = pr.Array(self.module, start_pos, array_type, parent) - """ - for i, value in enumerate(values): # TODO delete? - try: - keys[i] - except IndexError: - pass - """ - arr.values = values - arr.keys = keys - arr.type = array_type + # create an Array (-> needed for *args/**kwargs tuples/dicts) + arr = pr.Array(self.module, start_pos, array_type, parent) + arr.values = values + key_stmts = [] + for key in keys: + stmt = pr.Statement(self.module, 'XXX code', [], [], [], [], + start_pos, None) + stmt._commands = [key] + key_stmts.append(stmt) + arr.keys = key_stmts + arr.type = array_type - new_param._commands = [arr] - else: - new_param._commands = values + new_param._commands = [arr] name = copy.copy(param.get_name()) name.parent = new_param @@ -573,6 +569,7 @@ class Execution(Executable): keys = [] values = [] array_type = None + ignore_creation = False if commands[0] == '*': # *args param array_type = pr.Array.TUPLE @@ -591,12 +588,14 @@ class Execution(Executable): keys, values = zip(*non_matching_keys) else: # normal param - if value: + if value is not None: values = [value] else: if param.assignment_details: # No value: return the default values. - values = commands + ignore_creation = True + result.append(param.get_name()) + param.is_generated=True else: # If there is no assignment detail, that means there is # no assignment, just the result. Therefore nothing has @@ -605,7 +604,7 @@ class Execution(Executable): # Just ignore all the params that are without a key, after one # keyword argument was set. - if not keys_only or commands[0] == '**': + if not ignore_creation and (not keys_only or commands[0] == '**'): keys_used.add(str(key)) result.append(gen_param_name_copy(param, keys=keys, values=values, array_type=array_type)) @@ -653,7 +652,7 @@ class Execution(Executable): key_arr, op = stmt.assignment_details[0] # named parameter if key_arr and isinstance(key_arr[0], pr.Call): - yield op[0].name, stmt + yield key_arr[0].name, stmt else: yield None, stmt From fb6b3ce3422b87ebcec6846537f1bf8845025a77 Mon Sep 17 00:00:00 2001 From: David Halter Date: Sun, 10 Feb 2013 15:58:02 +0100 Subject: [PATCH 31/83] reintroduce parent and get_parent_until for er.Array, because an array is a builtin. always. --- jedi/evaluate_representation.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/jedi/evaluate_representation.py b/jedi/evaluate_representation.py index 3331a39c..606438d2 100644 --- a/jedi/evaluate_representation.py +++ b/jedi/evaluate_representation.py @@ -857,6 +857,13 @@ class Array(use_metaclass(cache.CachedMetaClass, pr.Base)): def get_contents(self): return self._array + @property + def parent(self): + return builtin.Builtin.scope + + def get_parent_until(self): + return builtin.Builtin.scope + def __getattr__(self, name): if name not in ['type', 'start_pos', 'get_only_subelement', 'parent', 'get_parent_until']: From a091ee92a450b764d29b07db385a3b83d7ce5c45 Mon Sep 17 00:00:00 2001 From: David Halter Date: Sun, 10 Feb 2013 16:12:30 +0100 Subject: [PATCH 32/83] fix dictionary index problems --- jedi/evaluate_representation.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/jedi/evaluate_representation.py b/jedi/evaluate_representation.py index 606438d2..d5717edc 100644 --- a/jedi/evaluate_representation.py +++ b/jedi/evaluate_representation.py @@ -815,26 +815,27 @@ class Array(use_metaclass(cache.CachedMetaClass, pr.Base)): return set(result) def get_exact_index_types(self, mixed_index): - """ Here the index is an int. Raises IndexError/KeyError """ + """ Here the index is an int/str. Raises IndexError/KeyError """ index = mixed_index if self.type == pr.Array.DICT: index = None for i, key_statement in enumerate(self._array.keys): # Because we only want the key to be a string. - key_elements = key_statement.get_commands() - if len(key_elements) == 1: + key_commands = key_statement.get_commands() + if len(key_commands) == 1: + key = key_commands[0] + key.get_code() try: - str_key = key_elements.get_code() + str_key = key.get_code() except AttributeError: - try: - str_key = key_elements[0].name - except AttributeError: - str_key = None + str_key = None if mixed_index == str_key: index = i break if index is None: raise KeyError('No key found in dictionary') + + # Can raise an IndexError values = [self._array.values[index]] return self._follow_values(values) From df17c98061ef11fa9c0d939cbe7b6b81cbd85dcb Mon Sep 17 00:00:00 2001 From: David Halter Date: Sun, 10 Feb 2013 16:59:51 +0100 Subject: [PATCH 33/83] fixed the whole var args iterator stuff --- jedi/evaluate_representation.py | 33 +++++++++++++++------------------ jedi/parsing_representation.py | 5 +++++ 2 files changed, 20 insertions(+), 18 deletions(-) diff --git a/jedi/evaluate_representation.py b/jedi/evaluate_representation.py index d5717edc..4c0dd311 100644 --- a/jedi/evaluate_representation.py +++ b/jedi/evaluate_representation.py @@ -627,25 +627,24 @@ class Execution(Executable): yield None, stmt # *args elif stmt.get_commands()[0] == '*': - arrays = evaluate.follow_call_list([stmt.token_list[1:]]) + arrays = evaluate.follow_call_list(stmt.get_commands()[1:]) # *args must be some sort of an array, otherwise -> ignore for array in arrays: - if hasattr(array, 'get_contents'): - for field in array.get_contents(): - yield None, field + for field_stmt in array: + yield None, field_stmt # **kwargs elif stmt.get_commands()[0] == '**': - arrays = evaluate.follow_call_list([stmt.token_list[1:]]) + arrays = evaluate.follow_call_list(stmt.get_commands()[1:]) for array in arrays: - if hasattr(array, 'get_contents'): - for key, field in array.get_contents(): - # Take the first index. - if isinstance(key, pr.Name): - name = key - else: - # `pr`.[Call|Function|Class] lookup. - name = key[0].name - yield name, field + for key_stmt, value_stmt in array.items(): + # first index, is the key if syntactically correct + call = key_stmt.get_commands()[0] + if type(call) == pr.Call: + yield call.name, value_stmt + else: + # `pr`.[Call|Function|Class] lookup. + # TODO remove? + yield key_stmt[0].name, value_stmt # Normal arguments (including key arguments). else: if stmt.assignment_details: @@ -855,9 +854,6 @@ class Array(use_metaclass(cache.CachedMetaClass, pr.Base)): names = scope.get_defined_names() return [ArrayMethod(n) for n in names] - def get_contents(self): - return self._array - @property def parent(self): return builtin.Builtin.scope @@ -867,7 +863,8 @@ class Array(use_metaclass(cache.CachedMetaClass, pr.Base)): def __getattr__(self, name): if name not in ['type', 'start_pos', 'get_only_subelement', 'parent', - 'get_parent_until']: + 'get_parent_until', 'items', + '__iter__', '__len__', '__getitem__']: raise AttributeError('Strange access on %s: %s.' % (self, name)) return getattr(self._array, name) diff --git a/jedi/parsing_representation.py b/jedi/parsing_representation.py index 7850d941..6e846c31 100644 --- a/jedi/parsing_representation.py +++ b/jedi/parsing_representation.py @@ -1073,6 +1073,11 @@ class Array(Call): raise TypeError('no dicts allowed') return iter(self.values) + def items(self): + if self.type != self.DICT: + raise TypeError('only dicts allowed') + return zip(self.keys, self.values) + def get_code(self): map = {self.NOARRAY: '(%s)', self.TUPLE: '(%s)', From 01c48593bf214651d72ef1483ec5781da84ce64a Mon Sep 17 00:00:00 2001 From: David Halter Date: Sun, 10 Feb 2013 21:15:04 +0100 Subject: [PATCH 34/83] er.Array is now also iterable --- jedi/evaluate_representation.py | 14 +++++++++++--- jedi/parsing_representation.py | 2 +- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/jedi/evaluate_representation.py b/jedi/evaluate_representation.py index 4c0dd311..371c76d0 100644 --- a/jedi/evaluate_representation.py +++ b/jedi/evaluate_representation.py @@ -630,7 +630,7 @@ class Execution(Executable): arrays = evaluate.follow_call_list(stmt.get_commands()[1:]) # *args must be some sort of an array, otherwise -> ignore for array in arrays: - for field_stmt in array: + for field_stmt in array: # yield from plz! yield None, field_stmt # **kwargs elif stmt.get_commands()[0] == '**': @@ -863,11 +863,19 @@ class Array(use_metaclass(cache.CachedMetaClass, pr.Base)): def __getattr__(self, name): if name not in ['type', 'start_pos', 'get_only_subelement', 'parent', - 'get_parent_until', 'items', - '__iter__', '__len__', '__getitem__']: + 'get_parent_until', 'items']: raise AttributeError('Strange access on %s: %s.' % (self, name)) return getattr(self._array, name) + def __getitem__(self): + return self._array.__getitem__() + + def __iter__(self): + return self._array.__iter__() + + def __len__(self): + return self._array.__len__() + def __repr__(self): return "" % (type(self).__name__, self._array) diff --git a/jedi/parsing_representation.py b/jedi/parsing_representation.py index 6e846c31..016690cf 100644 --- a/jedi/parsing_representation.py +++ b/jedi/parsing_representation.py @@ -767,7 +767,7 @@ class Statement(Simple): and not tok in ['>=', '<=', '==', '!='] def parse_array(token_iterator, array_type, start_pos, add_el=None): - arr = Array(self.module, start_pos, array_type) + arr = Array(self.module, start_pos, array_type, self) if add_el is not None: arr.add_statement(add_el) From 5722cd23828acad4c56c990756b650b934f39653 Mon Sep 17 00:00:00 2001 From: David Halter Date: Sun, 10 Feb 2013 22:39:21 +0100 Subject: [PATCH 35/83] fix parts of helpers.search_function_definition --- jedi/api.py | 3 +- jedi/helpers.py | 77 +++++++++++++++++++++++++++---------------------- 2 files changed, 43 insertions(+), 37 deletions(-) diff --git a/jedi/api.py b/jedi/api.py index e5f0d457..bd1077b6 100644 --- a/jedi/api.py +++ b/jedi/api.py @@ -355,9 +355,8 @@ class Script(object): if user_stmt is None \ or not isinstance(user_stmt, pr.Statement): return None, 0 - commands = helpers.fast_parent_copy(user_stmt.get_commands()) - call, index, stop = helpers.search_function_call(commands, self.pos) + call, index, stop = helpers.search_function_definition(user_stmt, self.pos) return call, index def check_cache(): diff --git a/jedi/helpers.py b/jedi/helpers.py index 3ac02694..f97b72f6 100644 --- a/jedi/helpers.py +++ b/jedi/helpers.py @@ -55,6 +55,9 @@ def fast_parent_copy(obj): elif isinstance(el, list): copied_list[i] = list_rec(el) return copied_list + + if isinstance(obj, list): + obj = tuple(obj) return recursion(obj) @@ -85,49 +88,53 @@ def array_for_pos(arr, pos): return result, check_arr_index(result, pos) -def search_function_call(arr, pos): +def search_function_definition(stmt, pos): """ - Returns the function Call that matches the position before `arr`. - This is somehow stupid, probably only the name of the function. + Returns the function Call that matches the position before. """ + def shorten(call): + return call + call = None stop = False - for sub in arr.values: + for command in stmt.get_commands(): call = None - for s in sub: - if isinstance(s, pr.Array): - new = search_function_call(s, pos) - if new[0] is not None: - call, index, stop = new - if stop: - return call, index, stop - elif isinstance(s, pr.Call): - start_s = s - # check parts of calls - while s is not None: - if s.start_pos >= pos: - return call, check_arr_index(arr, pos), stop - elif s.execution is not None: - end = s.execution.end_pos - if s.execution.start_pos < pos and \ - (None in end or pos < end): - c, index, stop = search_function_call( - s.execution, pos) - if stop: - return c, index, stop + command = 3 + if isinstance(command, pr.Array): + new = search_function_definition(command, pos) + if new[0] is not None: + call, index, stop = new + if stop: + return call, index, stop + elif isinstance(command, pr.Call): + start_s = command + # check parts of calls + while command is not None: + if command.start_pos >= pos: + return call, check_arr_index(command, pos), stop + elif command.execution is not None: + end = command.execution.end_pos + if command.execution.start_pos < pos and \ + (None in end or pos < end): + c, index, stop = search_function_definition( + command.execution, pos) + if stop: + return c, index, stop - # call should return without execution and - # next - reset = c or s - if reset.execution.type not in \ - [pr.Array.TUPLE, pr.Array.NOARRAY]: - return start_s, index, False + # call should return without execution and + # next + reset = c or command + if reset.execution.type not in \ + [pr.Array.TUPLE, pr.Array.NOARRAY]: + return start_s, index, False - reset.execution = None - reset.next = None - return c or start_s, index, True - s = s.next + call = fast_parent_copy(c or start_s) + reset.execution = None + reset.next = None + return call, index, True + command = command.next # The third return is just necessary for recursion inside, because # it needs to know when to stop iterating. + return None, 0, True # TODO remove return call, check_arr_index(arr, pos), stop From da4ad7fd3f860867fbbfbea27b743882a9878a9f Mon Sep 17 00:00:00 2001 From: David Halter Date: Sun, 10 Feb 2013 23:57:20 +0100 Subject: [PATCH 36/83] fix Name (instead of Param) as param key --- jedi/evaluate.py | 3 ++- jedi/evaluate_representation.py | 5 +++-- jedi/helpers.py | 3 --- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/jedi/evaluate.py b/jedi/evaluate.py index f1090a68..1adfbc3c 100644 --- a/jedi/evaluate.py +++ b/jedi/evaluate.py @@ -598,11 +598,12 @@ def follow_call_list(call_list, follow_array=False): # comprehensions result += follow_statement(stmt) else: - if isinstance(call, (pr.Lambda)): + if isinstance(call, pr.Lambda): result.append(er.Function(call)) # With things like params, these can also be functions... elif isinstance(call, (er.Function, er.Class, er.Instance, dynamic.ArrayInstance)): + # TODO this is just not very well readable -> fix, use pr.Base result.append(call) # The string tokens are just operations (+, -, etc.) elif not isinstance(call, (str, unicode)): diff --git a/jedi/evaluate_representation.py b/jedi/evaluate_representation.py index 371c76d0..db7f0957 100644 --- a/jedi/evaluate_representation.py +++ b/jedi/evaluate_representation.py @@ -305,7 +305,6 @@ class Function(use_metaclass(cache.CachedMetaClass, pr.Base)): """ Needed because of decorators. Decorators are evaluated here. """ - def __init__(self, func, is_decorated=False): """ This should not be called directly """ self.base_func = func @@ -639,7 +638,9 @@ class Execution(Executable): for key_stmt, value_stmt in array.items(): # first index, is the key if syntactically correct call = key_stmt.get_commands()[0] - if type(call) == pr.Call: + if isinstance(call, pr.Name): + yield call, value_stmt + elif type(call) == pr.Call: yield call.name, value_stmt else: # `pr`.[Call|Function|Class] lookup. diff --git a/jedi/helpers.py b/jedi/helpers.py index f97b72f6..04986497 100644 --- a/jedi/helpers.py +++ b/jedi/helpers.py @@ -55,9 +55,6 @@ def fast_parent_copy(obj): elif isinstance(el, list): copied_list[i] = list_rec(el) return copied_list - - if isinstance(obj, list): - obj = tuple(obj) return recursion(obj) From de5de416277a6c52550605aa57ab372e353c55c2 Mon Sep 17 00:00:00 2001 From: David Halter Date: Mon, 11 Feb 2013 00:06:31 +0100 Subject: [PATCH 37/83] fix last failing function tests --- jedi/evaluate_representation.py | 2 +- test/completion/functions.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/jedi/evaluate_representation.py b/jedi/evaluate_representation.py index db7f0957..10b74dfa 100644 --- a/jedi/evaluate_representation.py +++ b/jedi/evaluate_representation.py @@ -585,7 +585,7 @@ class Execution(Executable): array_type = pr.Array.DICT if non_matching_keys: keys, values = zip(*non_matching_keys) - else: + elif not keys_only: # normal param if value is not None: values = [value] diff --git a/test/completion/functions.py b/test/completion/functions.py index 501f00b1..387b8b0d 100644 --- a/test/completion/functions.py +++ b/test/completion/functions.py @@ -109,10 +109,10 @@ def func(a=1, b=''): return a, b exe = func(b=list, a=tuple) -#? tuple() +#? tuple exe[0] -#? list() +#? list exe[1] # ----------------- From f2a7788d66db67bce28e5065bd022bec24691ce0 Mon Sep 17 00:00:00 2001 From: David Halter Date: Mon, 11 Feb 2013 00:21:05 +0100 Subject: [PATCH 38/83] fix InstanceElement get_commands (doesn't raise an error anymore, but I don't know if it works) --- jedi/evaluate_representation.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/jedi/evaluate_representation.py b/jedi/evaluate_representation.py index 10b74dfa..e5f7412c 100644 --- a/jedi/evaluate_representation.py +++ b/jedi/evaluate_representation.py @@ -216,6 +216,8 @@ class InstanceElement(use_metaclass(cache.CachedMetaClass)): return func def get_commands(self): + """ + # TODO delete ? # Copy and modify the array. origin = self.var.get_commands() # Delete parent, because it isn't used anymore. @@ -224,6 +226,10 @@ class InstanceElement(use_metaclass(cache.CachedMetaClass)): self.is_class_var) new.parent_stmt = par return new + """ + # Copy and modify the array. + return [InstanceElement(self.instance, command, self.is_class_var) + for command in self.var.get_commands()] def __getattr__(self, name): return getattr(self.var, name) From cae38ed3d793d66772d813edd6bba7784f38ef70 Mon Sep 17 00:00:00 2001 From: David Halter Date: Mon, 11 Feb 2013 00:50:32 +0100 Subject: [PATCH 39/83] fix getattr/__getattr*__ --- jedi/evaluate.py | 9 ++++++--- jedi/evaluate_representation.py | 12 ++++++------ 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/jedi/evaluate.py b/jedi/evaluate.py index 1adfbc3c..ec8dee7f 100644 --- a/jedi/evaluate.py +++ b/jedi/evaluate.py @@ -414,9 +414,12 @@ def check_getattr(inst, name_str): """Checks for both __getattr__ and __getattribute__ methods""" result = [] # str is important to lose the NamePart! - name = pr.Call(str(name_str), pr.Call.STRING, (0, 0), inst) + module = builtin.Builtin.scope + name = pr.Call(module, str(name_str), pr.Call.STRING, (0, 0), inst) + stmt = pr.Statement(module, 'XXX code', [], [], [], [], (0, 0), None) + stmt._commands = [name] try: - result = inst.execute_subscope_by_name('__getattr__', [name]) + result = inst.execute_subscope_by_name('__getattr__', [stmt]) except KeyError: pass if not result: @@ -425,7 +428,7 @@ def check_getattr(inst, name_str): # could be practical and the jedi would return wrong types. If # you ever have something, let me know! try: - result = inst.execute_subscope_by_name('__getattribute__', [name]) + result = inst.execute_subscope_by_name('__getattribute__', [stmt]) except KeyError: pass return result diff --git a/jedi/evaluate_representation.py b/jedi/evaluate_representation.py index e5f7412c..9ba9366c 100644 --- a/jedi/evaluate_representation.py +++ b/jedi/evaluate_representation.py @@ -423,8 +423,10 @@ class Execution(Executable): debug.warning('getattr called without instance') continue - for name in names: - key = name.var_args.get_only_subelement() + for arr_name in names: + if len(arr_name.var_args) != 1: + debug.warning('jedi getattr is too simple') + key = arr_name.var_args[0] stmts += evaluate.follow_path(iter([key]), obj, self.base) return stmts @@ -507,7 +509,7 @@ class Execution(Executable): start_pos = self.var_args.start_pos else: parent = self.base - start_pos = None + start_pos = 0, 0 new_param = copy.copy(param) new_param.is_generated = True @@ -628,10 +630,8 @@ class Execution(Executable): def iterate(): # `var_args` is typically an Array, and not a list. for stmt in self.var_args: - if not isinstance(stmt, pr.Statement): - yield None, stmt # *args - elif stmt.get_commands()[0] == '*': + if stmt.get_commands()[0] == '*': arrays = evaluate.follow_call_list(stmt.get_commands()[1:]) # *args must be some sort of an array, otherwise -> ignore for array in arrays: From f423de1956548497386ce4b7785ee9c19740c6f6 Mon Sep 17 00:00:00 2001 From: David Halter Date: Mon, 11 Feb 2013 01:08:45 +0100 Subject: [PATCH 40/83] fix some descriptor methods --- jedi/evaluate.py | 6 ++---- jedi/evaluate_representation.py | 16 +++++++++++++--- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/jedi/evaluate.py b/jedi/evaluate.py index ec8dee7f..91e1bf96 100644 --- a/jedi/evaluate.py +++ b/jedi/evaluate.py @@ -416,10 +416,8 @@ def check_getattr(inst, name_str): # str is important to lose the NamePart! module = builtin.Builtin.scope name = pr.Call(module, str(name_str), pr.Call.STRING, (0, 0), inst) - stmt = pr.Statement(module, 'XXX code', [], [], [], [], (0, 0), None) - stmt._commands = [name] try: - result = inst.execute_subscope_by_name('__getattr__', [stmt]) + result = inst.execute_subscope_by_name('__getattr__', [name]) except KeyError: pass if not result: @@ -428,7 +426,7 @@ def check_getattr(inst, name_str): # could be practical and the jedi would return wrong types. If # you ever have something, let me know! try: - result = inst.execute_subscope_by_name('__getattribute__', [stmt]) + result = inst.execute_subscope_by_name('__getattribute__', [name]) except KeyError: pass return result diff --git a/jedi/evaluate_representation.py b/jedi/evaluate_representation.py index 9ba9366c..8b991594 100644 --- a/jedi/evaluate_representation.py +++ b/jedi/evaluate_representation.py @@ -298,9 +298,9 @@ class Class(use_metaclass(cache.CachedMetaClass, pr.Base)): return self.base.name def __getattr__(self, name): - if name not in ['start_pos', 'end_pos', 'parent', 'subscopes', - 'get_imports', 'get_parent_until', 'docstr', 'asserts']: - raise AttributeError("Don't touch this (%s)!" % name) + if name not in ['start_pos', 'end_pos', 'parent', 'asserts', 'docstr', + 'get_imports', 'get_parent_until', 'get_code', 'subscopes']: + raise AttributeError("Don't touch this: %s of %s !" % (name, self)) return getattr(self.base, name) def __repr__(self): @@ -630,6 +630,16 @@ class Execution(Executable): def iterate(): # `var_args` is typically an Array, and not a list. for stmt in self.var_args: + if not isinstance(stmt, pr.Statement): + if stmt is None: + yield None, None + continue + old = stmt + # generate a statement if it's not already one. + module = builtin.Builtin.scope + stmt = pr.Statement(module, 'XXX code', [], [], [], [], (0, 0), None) + stmt._commands = [old] + # *args if stmt.get_commands()[0] == '*': arrays = evaluate.follow_call_list(stmt.get_commands()[1:]) From 52b32a01c11ecc24c776ad0f7a018ca054ba2784 Mon Sep 17 00:00:00 2001 From: David Halter Date: Mon, 11 Feb 2013 01:21:51 +0100 Subject: [PATCH 41/83] move default arguments [] to (), because mutable may be dangerous (especially in a recursive environment --- jedi/api.py | 2 +- jedi/evaluate.py | 2 +- jedi/evaluate_representation.py | 16 ++++++++-------- jedi/imports.py | 2 +- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/jedi/api.py b/jedi/api.py index bd1077b6..26edaf5a 100644 --- a/jedi/api.py +++ b/jedi/api.py @@ -300,7 +300,7 @@ class Script(object): definitions = [user_stmt] return definitions, search_name - def related_names(self, additional_module_paths=[]): + def related_names(self, additional_module_paths=()): """ Return :class:`api_classes.RelatedName` objects, which contain all names that point to the definition of the name under the cursor. This diff --git a/jedi/evaluate.py b/jedi/evaluate.py index 91e1bf96..a82d26ac 100644 --- a/jedi/evaluate.py +++ b/jedi/evaluate.py @@ -520,7 +520,7 @@ def assign_tuples(tup, results, seek_name): @recursion.RecursionDecorator -@cache.memoize_default(default=[]) +@cache.memoize_default(default=()) def follow_statement(stmt, seek_name=None): """ The starting point of the completion. A statement always owns a call list, diff --git a/jedi/evaluate_representation.py b/jedi/evaluate_representation.py index 8b991594..7847395d 100644 --- a/jedi/evaluate_representation.py +++ b/jedi/evaluate_representation.py @@ -39,7 +39,7 @@ class Executable(pr.Base): An instance is also an executable - because __init__ is called :param var_args: The param input array, consist of `pr.Array` or list. """ - def __init__(self, base, var_args=[]): + def __init__(self, base, var_args=()): self.base = base self.var_args = var_args @@ -53,7 +53,7 @@ class Executable(pr.Base): class Instance(use_metaclass(cache.CachedMetaClass, Executable)): """ This class is used to evaluate instances. """ - def __init__(self, base, var_args=[]): + def __init__(self, base, var_args=()): super(Instance, self).__init__(base, var_args) if str(base.name) in ['list', 'set'] \ and builtin.Builtin.scope == base.get_parent_until(): @@ -122,7 +122,7 @@ class Instance(use_metaclass(cache.CachedMetaClass, Executable)): sub = self.base.get_subscope_by_name(name) return InstanceElement(self, sub, True) - def execute_subscope_by_name(self, name, args=[]): + def execute_subscope_by_name(self, name, args=()): method = self.get_subscope_by_name(name) return Execution(method, args).get_return_types() @@ -249,7 +249,7 @@ class Class(use_metaclass(cache.CachedMetaClass, pr.Base)): def __init__(self, base): self.base = base - @cache.memoize_default(default=[]) + @cache.memoize_default(default=()) def get_super_classes(self): supers = [] # TODO care for mro stuff (multiple super classes). @@ -265,7 +265,7 @@ class Class(use_metaclass(cache.CachedMetaClass, pr.Base)): supers += evaluate.find_name(builtin.Builtin.scope, 'object') return supers - @cache.memoize_default(default=[]) + @cache.memoize_default(default=()) def get_defined_names(self): def in_iterable(name, iterable): """ checks if the name is in the variable 'iterable'. """ @@ -399,7 +399,7 @@ class Execution(Executable): else: return [stmt] # just some arbitrary object - @cache.memoize_default(default=[]) + @cache.memoize_default(default=()) @recursion.ExecutionRecursionDecorator def get_return_types(self, evaluate_generator=False): """ Get the return types of a function. """ @@ -490,7 +490,7 @@ class Execution(Executable): stmts += evaluate.follow_statement(r) return stmts - @cache.memoize_default(default=[]) + @cache.memoize_default(default=()) def get_params(self): """ This returns the params for an Execution/Instance and is injected as a @@ -498,7 +498,7 @@ class Execution(Executable): This needs to be here, because Instance can have __init__ functions, which act the same way as normal functions. """ - def gen_param_name_copy(param, keys=[], values=[], array_type=None): + def gen_param_name_copy(param, keys=(), values=(), array_type=None): """ Create a param with the original scope (of varargs) as parent. """ diff --git a/jedi/imports.py b/jedi/imports.py index a7739e27..19c13e9b 100644 --- a/jedi/imports.py +++ b/jedi/imports.py @@ -318,7 +318,7 @@ def strip_imports(scopes): @cache.cache_star_import -def remove_star_imports(scope, ignored_modules=[]): +def remove_star_imports(scope, ignored_modules=()): """ Check a module for star imports: >>> from module import * From cda84dc92ac83ea4ece8a5b8b01a707a474a94fa Mon Sep 17 00:00:00 2001 From: David Halter Date: Thu, 14 Feb 2013 20:54:26 +0400 Subject: [PATCH 42/83] add an empty IsScope class to match all scopes --- jedi/evaluate.py | 2 +- jedi/evaluate_representation.py | 6 +++--- jedi/parsing_representation.py | 6 +++++- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/jedi/evaluate.py b/jedi/evaluate.py index a82d26ac..3bf5f926 100644 --- a/jedi/evaluate.py +++ b/jedi/evaluate.py @@ -634,7 +634,7 @@ def follow_call_list(call_list, follow_array=False): def follow_call(call): """Follow a call is following a function, variable, string, etc.""" path = call.generate_call_path() - scope = call.get_parent_until((pr.Scope, er.Execution)) + scope = call.get_parent_until(pr.IsScope) return follow_call_path(path, scope, call.start_pos) diff --git a/jedi/evaluate_representation.py b/jedi/evaluate_representation.py index 7847395d..f38114c5 100644 --- a/jedi/evaluate_representation.py +++ b/jedi/evaluate_representation.py @@ -34,7 +34,7 @@ class DecoratorNotFound(LookupError): pass -class Executable(pr.Base): +class Executable(pr.IsScope): """ An instance is also an executable - because __init__ is called :param var_args: The param input array, consist of `pr.Array` or list. @@ -241,7 +241,7 @@ class InstanceElement(use_metaclass(cache.CachedMetaClass)): return "<%s of %s>" % (type(self).__name__, self.var) -class Class(use_metaclass(cache.CachedMetaClass, pr.Base)): +class Class(use_metaclass(cache.CachedMetaClass, pr.IsScope)): """ This class is not only important to extend `pr.Class`, it is also a important for descriptors (if the descriptor methods are evaluated or not). @@ -307,7 +307,7 @@ class Class(use_metaclass(cache.CachedMetaClass, pr.Base)): return "" % (type(self).__name__, self.base) -class Function(use_metaclass(cache.CachedMetaClass, pr.Base)): +class Function(use_metaclass(cache.CachedMetaClass, pr.IsScope)): """ Needed because of decorators. Decorators are evaluated here. """ diff --git a/jedi/parsing_representation.py b/jedi/parsing_representation.py index 016690cf..23396671 100644 --- a/jedi/parsing_representation.py +++ b/jedi/parsing_representation.py @@ -97,7 +97,11 @@ class Simple(Base): (type(self).__name__, code, self.start_pos[0]) -class Scope(Simple): +class IsScope(Base): + pass + + +class Scope(Simple, IsScope): """ Super class for the parser tree, which represents the state of a python text file. From fb0ea354acd3efc2a58bddf8d14e6428c2dcd7a1 Mon Sep 17 00:00:00 2001 From: David Halter Date: Thu, 14 Feb 2013 20:57:55 +0400 Subject: [PATCH 43/83] fix a dynamic search param problem --- jedi/dynamic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jedi/dynamic.py b/jedi/dynamic.py index aecd18f6..a59da2ac 100644 --- a/jedi/dynamic.py +++ b/jedi/dynamic.py @@ -161,7 +161,7 @@ def search_params(param): else: commands = param.get_commands() offset = 1 if commands[0] in ['*', '**'] else 0 - param_name = str(commands[0][offset].name) + param_name = str(commands[offset].name) # add the listener listener = ParamListener() From 8fd39109584fce8d03b0f0d8caac1f3df6f986bd Mon Sep 17 00:00:00 2001 From: David Halter Date: Sat, 16 Feb 2013 23:38:07 +0430 Subject: [PATCH 44/83] fix a get_code problem for instances/statements, which is not needed there, because get_code should only be called on pure parsing_representation objects --- jedi/parsing_representation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jedi/parsing_representation.py b/jedi/parsing_representation.py index 23396671..43b82d5d 100644 --- a/jedi/parsing_representation.py +++ b/jedi/parsing_representation.py @@ -715,7 +715,7 @@ class Statement(Simple): def get_code(self, new_line=True): def assemble(command_list, assignment=None): - pieces = [c.get_code() if isinstance(c, Base) else c + pieces = [c.get_code() if isinstance(c, Simple) else unicode(c) for c in command_list] if assignment is None: return ''.join(pieces) From 89a2cb15d08263c2c3dc5c7adbfecd382a24ab87 Mon Sep 17 00:00:00 2001 From: David Halter Date: Sun, 17 Feb 2013 00:14:30 +0430 Subject: [PATCH 45/83] fix problems with positions in definitions with the same name on the same line --- jedi/evaluate.py | 8 ++++++-- jedi/parsing_representation.py | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/jedi/evaluate.py b/jedi/evaluate.py index 3bf5f926..127f77b0 100644 --- a/jedi/evaluate.py +++ b/jedi/evaluate.py @@ -634,8 +634,12 @@ def follow_call_list(call_list, follow_array=False): def follow_call(call): """Follow a call is following a function, variable, string, etc.""" path = call.generate_call_path() - scope = call.get_parent_until(pr.IsScope) - return follow_call_path(path, scope, call.start_pos) + + # find the statement of the Scope + s = call + while not s.parent.isinstance(pr.IsScope): + s = s.parent + return follow_call_path(path, s.parent, s.start_pos) def follow_call_path(path, scope, position): diff --git a/jedi/parsing_representation.py b/jedi/parsing_representation.py index 43b82d5d..cda636fd 100644 --- a/jedi/parsing_representation.py +++ b/jedi/parsing_representation.py @@ -253,7 +253,7 @@ class Scope(Simple, IsScope): self.start_pos[0], self.end_pos[0]) -class Module(object): +class Module(IsScope): """ For isinstance checks. fast_parser.Module also inherits from this. """ pass From 1366f5fa6103cd4fb643463eb97d5f3eb9cffe76 Mon Sep 17 00:00:00 2001 From: David Halter Date: Sun, 17 Feb 2013 00:23:43 +0430 Subject: [PATCH 46/83] made ListComprehensions behave more llike typical 'pr' objects --- jedi/parsing.py | 5 ++--- jedi/parsing_representation.py | 15 +++++++++++++-- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/jedi/parsing.py b/jedi/parsing.py index a38b8746..b8419c26 100644 --- a/jedi/parsing.py +++ b/jedi/parsing.py @@ -439,9 +439,8 @@ class Parser(object): st = pr.Statement(self.module, src, [], [], [], toks, first_pos, self.end_pos) - for s in [st, middle, in_clause]: - s.parent = self.scope - tok = pr.ListComprehension(st, middle, in_clause) + tok = pr.ListComprehension(st, middle, in_clause, + self.scope) tok_list.append(tok) if list_comp: string = '' diff --git a/jedi/parsing_representation.py b/jedi/parsing_representation.py index cda636fd..b136fd4a 100644 --- a/jedi/parsing_representation.py +++ b/jedi/parsing_representation.py @@ -1166,12 +1166,23 @@ class Name(Simple): return len(self.names) -class ListComprehension(object): +class ListComprehension(Base): """ Helper class for list comprehensions """ - def __init__(self, stmt, middle, input): + def __init__(self, stmt, middle, input, parent): self.stmt = stmt self.middle = middle self.input = input + for s in [stmt, middle, input]: + s.parent = self + self.parent = parent + + @property + def start_pos(self): + return self.stmt.start_pos + + @property + def end_pos(self): + return self.stmt.end_pos def __repr__(self): return "<%s: %s>" % \ From 79216f189f805a9bdd285bedb4efd847e747dc81 Mon Sep 17 00:00:00 2001 From: David Halter Date: Sun, 17 Feb 2013 00:30:44 +0430 Subject: [PATCH 47/83] moved parsing.Simple.module to _sub_module --- jedi/evaluate_representation.py | 6 +++--- jedi/helpers.py | 2 +- jedi/parsing_representation.py | 16 ++++++++-------- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/jedi/evaluate_representation.py b/jedi/evaluate_representation.py index f38114c5..5f3fb896 100644 --- a/jedi/evaluate_representation.py +++ b/jedi/evaluate_representation.py @@ -517,11 +517,11 @@ class Execution(Executable): new_param.parent = parent # create an Array (-> needed for *args/**kwargs tuples/dicts) - arr = pr.Array(self.module, start_pos, array_type, parent) + arr = pr.Array(self._sub_module, start_pos, array_type, parent) arr.values = values key_stmts = [] for key in keys: - stmt = pr.Statement(self.module, 'XXX code', [], [], [], [], + stmt = pr.Statement(self._sub_module, 'XXX code', [], [], [], [], start_pos, None) stmt._commands = [key] key_stmts.append(stmt) @@ -709,7 +709,7 @@ class Execution(Executable): raise common.MultiLevelAttributeError(sys.exc_info()) def __getattr__(self, name): - if name not in ['start_pos', 'end_pos', 'imports', 'module']: + if name not in ['start_pos', 'end_pos', 'imports', '_sub_module']: raise AttributeError('Tried to access %s: %s. Why?' % (name, self)) return getattr(self.base, name) diff --git a/jedi/helpers.py b/jedi/helpers.py index 04986497..ea331503 100644 --- a/jedi/helpers.py +++ b/jedi/helpers.py @@ -39,7 +39,7 @@ def fast_parent_copy(obj): setattr(new_obj, key, new_elements[value]) except KeyError: pass - elif key in ['parent_function', 'use_as_parent', 'module']: + elif key in ['parent_function', 'use_as_parent', '_sub_module']: continue elif isinstance(value, list): setattr(new_obj, key, list_rec(value)) diff --git a/jedi/parsing_representation.py b/jedi/parsing_representation.py index b136fd4a..4baf4534 100644 --- a/jedi/parsing_representation.py +++ b/jedi/parsing_representation.py @@ -49,10 +49,10 @@ class Simple(Base): The super class for Scope, Import, Name and Statement. Every object in the parser tree inherits from this class. """ - __slots__ = ('parent', 'module', '_start_pos', 'use_as_parent', '_end_pos') + __slots__ = ('parent', '_sub_module', '_start_pos', 'use_as_parent', '_end_pos') def __init__(self, module, start_pos, end_pos=(None, None)): - self.module = module + self._sub_module = module self._start_pos = start_pos self._end_pos = end_pos @@ -62,7 +62,7 @@ class Simple(Base): @property def start_pos(self): - return self.module.line_offset + self._start_pos[0], self._start_pos[1] + return self._sub_module.line_offset + self._start_pos[0], self._start_pos[1] @start_pos.setter def start_pos(self, value): @@ -72,7 +72,7 @@ class Simple(Base): def end_pos(self): if None in self._end_pos: return self._end_pos - return self.module.line_offset + self._end_pos[0], self._end_pos[1] + return self._sub_module.line_offset + self._end_pos[0], self._end_pos[1] @end_pos.setter def end_pos(self, value): @@ -621,7 +621,7 @@ class Import(Simple): return [self.alias] if len(self.namespace) > 1: o = self.namespace - n = Name(self.module, [(o.names[0], o.start_pos)], o.start_pos, + n = Name(self._sub_module, [(o.names[0], o.start_pos)], o.start_pos, o.end_pos, parent=o.parent) return [n] else: @@ -771,7 +771,7 @@ class Statement(Simple): and not tok in ['>=', '<=', '==', '!='] def parse_array(token_iterator, array_type, start_pos, add_el=None): - arr = Array(self.module, start_pos, array_type, self) + arr = Array(self._sub_module, start_pos, array_type, self) if add_el is not None: arr.add_statement(add_el) @@ -841,7 +841,7 @@ class Statement(Simple): if not token_list: return None, tok - statement = Statement(self.module, "XXX" + self.code, [], [], [], + statement = Statement(self._sub_module, "XXX" + self.code, [], [], [], token_list, start_pos, end_pos) statement.parent = self.parent return statement, tok @@ -883,7 +883,7 @@ class Statement(Simple): elif token_type == tokenize.NUMBER: c_type = Call.NUMBER - call = Call(self.module, tok, c_type, start_pos, self) + call = Call(self._sub_module, tok, c_type, start_pos, self) if is_chain: result[-1].set_next(call) else: From 779ce6509d558f4e5c8e4e547d82679276e3fb02 Mon Sep 17 00:00:00 2001 From: David Halter Date: Sun, 17 Feb 2013 22:15:11 +0430 Subject: [PATCH 48/83] first dynamic array work --- jedi/dynamic.py | 34 ++++++++++++++++++++-------------- jedi/recursion.py | 8 +++++--- 2 files changed, 25 insertions(+), 17 deletions(-) diff --git a/jedi/dynamic.py b/jedi/dynamic.py index a59da2ac..cd38ec60 100644 --- a/jedi/dynamic.py +++ b/jedi/dynamic.py @@ -267,6 +267,8 @@ def _check_array_additions(compare_array, module, is_list): stmt = element._array.parent else: # must be instance + if isinstance(element.var_args, list): + return None # TODO check if this is ok stmt = element.var_args.parent if isinstance(stmt, er.InstanceElement): stop_classes = list(stop_classes) + [er.Function] @@ -278,6 +280,9 @@ def _check_array_additions(compare_array, module, is_list): search_names = ['append', 'extend', 'insert'] if is_list else \ ['add', 'update'] comp_arr_parent = get_execution_parent(compare_array, er.Execution) + if comp_arr_parent is None: + return [] # TODO check if this is ok + possible_stmts = [] res = [] for n in search_names: @@ -334,23 +339,24 @@ class ArrayInstance(pr.Base): lists/sets are too complicated too handle that. """ items = [] - for array in evaluate.follow_call_list(self.var_args): - if isinstance(array, er.Instance) and len(array.var_args): - temp = array.var_args[0][0] - if isinstance(temp, ArrayInstance): - # prevent recursions - # TODO compare Modules - if self.var_args.start_pos != temp.var_args.start_pos: - items += temp.iter_content() - else: - debug.warning('ArrayInstance recursion', self.var_args) - continue - items += evaluate.get_iterator_types([array]) + for stmt in self.var_args: + for typ in evaluate.follow_statement(stmt): + if isinstance(typ, er.Instance) and len(typ.var_args): + array = typ.var_args[0][0] + if isinstance(array, ArrayInstance): + # prevent recursions + # TODO compare Modules + if self.var_args.start_pos != array.var_args.start_pos: + items += array.iter_content() + else: + debug.warning('ArrayInstance recursion', self.var_args) + continue + items += evaluate.get_iterator_types([typ]) - if self.var_args.parent_stmt is None: + if self.var_args.parent is None: return [] # generated var_args should not be checked for arrays - module = self.var_args.parent_stmt.get_parent_until() + module = self.var_args.get_parent_until() is_list = str(self.instance.name) == 'list' items += _check_array_additions(self.instance, module, is_list) return items diff --git a/jedi/recursion.py b/jedi/recursion.py index 70c9aaa4..cdb1eff8 100644 --- a/jedi/recursion.py +++ b/jedi/recursion.py @@ -34,8 +34,10 @@ class RecursionDecorator(object): def push_stmt(self, stmt): self.current = RecursionNode(stmt, self.current) - if self._check_recursion(): - debug.warning('catched recursion', stmt, stmt.start_pos) + check = self._check_recursion() + if check: + debug.warning('catched stmt recursion: %s against %s @%s' + % (stmt, stmt.start_pos, check.stmt)) self.pop_stmt() return True return False @@ -51,7 +53,7 @@ class RecursionDecorator(object): while True: test = test.parent if self.current == test: - return True + return test if not test: return False From c67b9986a68143a74fa6d7aac1cfddbc3a154990 Mon Sep 17 00:00:00 2001 From: David Halter Date: Sun, 17 Feb 2013 22:48:41 +0430 Subject: [PATCH 49/83] some minor fixes / inits to inputs refactoring --- jedi/dynamic.py | 9 +++++---- jedi/evaluate.py | 7 ++++--- jedi/modules.py | 2 ++ jedi/parsing.py | 10 +++++----- jedi/parsing_representation.py | 20 ++++++++++---------- 5 files changed, 26 insertions(+), 22 deletions(-) diff --git a/jedi/dynamic.py b/jedi/dynamic.py index cd38ec60..9d5ba14e 100644 --- a/jedi/dynamic.py +++ b/jedi/dynamic.py @@ -194,9 +194,10 @@ def check_array_additions(array): def _scan_array(arr, search_name): """ Returns the function Call that match search_name in an Array. """ + print arr result = [] - for sub in arr: - for s in sub: + for stmt in arr: + for s in stmt.get_commands(): if isinstance(s, pr.Array): result += _scan_array(s, search_name) elif isinstance(s, pr.Call): @@ -461,8 +462,8 @@ def check_flow_information(flow, search_name, pos): break if isinstance(flow, pr.Flow) and not result: - if flow.command in ['if', 'while'] and len(flow.inits) == 1: - result = check_statement_information(flow.inits[0], search_name) + if flow.command in ['if', 'while'] and len(flow.inputs) == 1: + result = check_statement_information(flow.inputs[0], search_name) return result diff --git a/jedi/evaluate.py b/jedi/evaluate.py index 127f77b0..b860f60e 100644 --- a/jedi/evaluate.py +++ b/jedi/evaluate.py @@ -248,12 +248,13 @@ def find_name(scope, name_str, position=None, search_global=False, def handle_for_loops(loop): # Take the first statement (for has always only # one, remember `in`). And follow it. - if not len(loop.inits): + if not loop.inputs: return [] - result = get_iterator_types(follow_statement(loop.inits[0])) + result = get_iterator_types(follow_statement(loop.inputs[0])) if len(loop.set_vars) > 1: commands = loop.set_stmt.get_commands() - result = assign_tuples(commands, result, name_str) + # loops with loop.set_vars > 0 only have one command + result = assign_tuples(commands[0], result, name_str) return result def process(name): diff --git a/jedi/modules.py b/jedi/modules.py index b40d3f30..b069f986 100644 --- a/jedi/modules.py +++ b/jedi/modules.py @@ -311,6 +311,8 @@ def sys_path_with_modifications(module): sys_path = list(get_sys_path()) # copy for p in possible_stmts: + if not isinstance(p, pr.Statement): + continue commands = p.get_commands() assert len(commands) == 1 call = commands[0] diff --git a/jedi/parsing.py b/jedi/parsing.py index b8419c26..3dfcc5f8 100644 --- a/jedi/parsing.py +++ b/jedi/parsing.py @@ -658,8 +658,8 @@ class Parser(object): command = tok if command in ['except', 'with']: added_breaks.append(',') - # multiple statements because of with - inits = [] + # multiple inputs because of with + inputs = [] first = True while first or command == 'with' \ and tok not in [':', '\n']: @@ -673,11 +673,11 @@ class Parser(object): statement.set_vars.append(n) statement.code += ',' + n.get_code() if statement: - inits.append(statement) + inputs.append(statement) first = False if tok == ':': - f = pr.Flow(self.module, command, inits, first_pos) + f = pr.Flow(self.module, command, inputs, first_pos) if command in extended_flow: # the last statement has to be another part of # the flow statement, because a dedent releases the @@ -691,7 +691,7 @@ class Parser(object): s = self.scope.add_statement(f) self.scope = s else: - for i in inits: + for i in inputs: i.parent = use_as_parent_scope debug.warning('syntax err, flow started @%s', self.start_pos[0]) diff --git a/jedi/parsing_representation.py b/jedi/parsing_representation.py index 4baf4534..67532b32 100644 --- a/jedi/parsing_representation.py +++ b/jedi/parsing_representation.py @@ -457,21 +457,21 @@ class Flow(Scope): :param command: The flow command, if, while, else, etc. :type command: str - :param inits: The initializations of a flow -> while 'statement'. - :type inits: list(Statement) + :param inputs: The initializations of a flow -> while 'statement'. + :type inputs: list(Statement) :param start_pos: Position (line, column) of the Flow statement. :type start_pos: tuple(int, int) :param set_vars: Local variables used in the for loop (only there). :type set_vars: list """ - def __init__(self, module, command, inits, start_pos, set_vars=None): + def __init__(self, module, command, inputs, start_pos, set_vars=None): self.next = None self.command = command super(Flow, self).__init__(module, start_pos) self._parent = None # These have to be statements, because of with, which takes multiple. - self.inits = inits - for s in inits: + self.inputs = inputs + for s in inputs: s.parent = self.use_as_parent if set_vars is None: self.set_vars = [] @@ -493,7 +493,7 @@ class Flow(Scope): def get_code(self, first_indent=False, indention=' '): stmts = [] - for s in self.inits: + for s in self.inputs: stmts.append(s.get_code(new_line=False)) stmt = ', '.join(stmts) string = "%s %s:\n" % (self.command, stmt) @@ -512,7 +512,7 @@ class Flow(Scope): """ if is_internal_call: n = list(self.set_vars) - for s in self.inits: + for s in self.inputs: n += s.set_vars if self.next: n += self.next.get_set_vars(is_internal_call) @@ -541,8 +541,8 @@ class ForFlow(Flow): """ Used for the for loop, because there are two statement parts. """ - def __init__(self, module, inits, start_pos, set_stmt, is_list_comp=False): - super(ForFlow, self).__init__(module, 'for', inits, start_pos, + def __init__(self, module, inputs, start_pos, set_stmt, is_list_comp=False): + super(ForFlow, self).__init__(module, 'for', inputs, start_pos, set_stmt.used_vars) self.set_stmt = set_stmt self.is_list_comp = is_list_comp @@ -550,7 +550,7 @@ class ForFlow(Flow): def get_code(self, first_indent=False, indention=" " * 4): vars = ",".join(x.get_code() for x in self.set_vars) stmts = [] - for s in self.inits: + for s in self.inputs: stmts.append(s.get_code(new_line=False)) stmt = ', '.join(stmts) s = "for %s in %s:\n" % (vars, stmt) From 108f3956709988959fc27e885ec1720acb887aaa Mon Sep 17 00:00:00 2001 From: David Halter Date: Sun, 17 Feb 2013 22:58:13 +0430 Subject: [PATCH 50/83] fixed ordering.py tests --- jedi/dynamic.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/jedi/dynamic.py b/jedi/dynamic.py index 9d5ba14e..08982855 100644 --- a/jedi/dynamic.py +++ b/jedi/dynamic.py @@ -133,7 +133,9 @@ def search_params(param): for stmt in possible_stmts: if not isinstance(stmt, pr.Import): - calls = _scan_array(stmt.get_commands(), func_name) + arr = [c for c in stmt.get_commands() + if isinstance(c, pr.Statement)] + calls = _scan_array(arr, func_name) for c in calls: # no execution means that params cannot be set call_path = c.generate_call_path() @@ -157,7 +159,8 @@ def search_params(param): # get the param name if param.assignment_details: - commands = param.assignment_details[0] + # first assignment details, others would be a syntax error + commands, op = param.assignment_details[0] else: commands = param.get_commands() offset = 1 if commands[0] in ['*', '**'] else 0 @@ -194,7 +197,6 @@ def check_array_additions(array): def _scan_array(arr, search_name): """ Returns the function Call that match search_name in an Array. """ - print arr result = [] for stmt in arr: for s in stmt.get_commands(): From 4e102e25e69515786fae51b74d4e5935d165aca3 Mon Sep 17 00:00:00 2001 From: David Halter Date: Sun, 17 Feb 2013 23:23:57 +0430 Subject: [PATCH 51/83] fix some of the dynamic bugs --- jedi/dynamic.py | 69 ++++++++++++++++++++++++++++-------------------- jedi/evaluate.py | 3 +-- 2 files changed, 41 insertions(+), 31 deletions(-) diff --git a/jedi/dynamic.py b/jedi/dynamic.py index 08982855..f0301d81 100644 --- a/jedi/dynamic.py +++ b/jedi/dynamic.py @@ -21,7 +21,6 @@ import parsing_representation as pr import evaluate_representation as er import modules import evaluate -import helpers import settings import debug import imports @@ -133,9 +132,7 @@ def search_params(param): for stmt in possible_stmts: if not isinstance(stmt, pr.Import): - arr = [c for c in stmt.get_commands() - if isinstance(c, pr.Statement)] - calls = _scan_array(arr, func_name) + calls = _scan_statement(stmt, func_name) for c in calls: # no execution means that params cannot be set call_path = c.generate_call_path() @@ -195,23 +192,37 @@ def check_array_additions(array): return res -def _scan_array(arr, search_name): +def _scan_statement(stmt, search_name, assignment_details=False): """ Returns the function Call that match search_name in an Array. """ - result = [] - for stmt in arr: - for s in stmt.get_commands(): - if isinstance(s, pr.Array): - result += _scan_array(s, search_name) - elif isinstance(s, pr.Call): - s_new = s - while s_new is not None: - n = s_new.name - if isinstance(n, pr.Name) and search_name in n.names: - result.append(s) + def scan_array(arr, search_name): + result = [] + if arr.type == pr.Array.DICT: + for key_stmt, value_stmt in arr.items(): + result += _scan_statement(key_stmt, search_name) + result += _scan_statement(value_stmt, search_name) + else: + for stmt in arr: + result += _scan_statement(stmt, search_name) + return result - if s_new.execution is not None: - result += _scan_array(s_new.execution, search_name) - s_new = s_new.next + result = [] + for c in stmt.get_commands(): + if isinstance(c, pr.Array): + result += scan_array(c, search_name) + elif isinstance(c, pr.Call): + s_new = c + while s_new is not None: + n = s_new.name + if isinstance(n, pr.Name) and search_name in n.names: + result.append(c) + + if s_new.execution is not None: + result += scan_array(s_new.execution, search_name) + s_new = s_new.next + + if assignment_details: + for stmt, op in stmt.assignment_details: + result += _scan_statement(stmt, search_name) return result @@ -241,7 +252,7 @@ def _check_array_additions(compare_array, module, is_list): backtrack_path = iter(call_path[:separate_index]) position = c.start_pos - scope = c.parent_stmt.parent + scope = c.get_parent_until(pr.IsScope) found = evaluate.follow_call_path(backtrack_path, scope, position) if not compare_array in found: @@ -251,16 +262,18 @@ def _check_array_additions(compare_array, module, is_list): if not params.values: continue # no params: just ignore it if add_name in ['append', 'add']: - result += evaluate.follow_call_list(params) + for param in params: + result += evaluate.follow_call_list(param.get_commands()) elif add_name in ['insert']: try: second_param = params[1] except IndexError: continue else: - result += evaluate.follow_call_list([second_param]) + result += evaluate.follow_call_list(second_param.get_comands()) elif add_name in ['extend', 'update']: - iterators = evaluate.follow_call_list(params) + for param in params: + iterators = evaluate.follow_call_list(param.get_commands()) result += evaluate.get_iterator_types(iterators) return result @@ -311,7 +324,7 @@ def _check_array_additions(compare_array, module, is_list): if evaluate.follow_statement.push_stmt(stmt): # check recursion continue - res += check_calls(_scan_array(stmt.get_commands(), n), n) + res += check_calls(_scan_statement(stmt, n), n) evaluate.follow_statement.pop_stmt() # reset settings settings.dynamic_params_for_other_modules = temp_param_add @@ -345,7 +358,7 @@ class ArrayInstance(pr.Base): for stmt in self.var_args: for typ in evaluate.follow_statement(stmt): if isinstance(typ, er.Instance) and len(typ.var_args): - array = typ.var_args[0][0] + array = typ.var_args[0] if isinstance(array, ArrayInstance): # prevent recursions # TODO compare Modules @@ -425,10 +438,8 @@ def related_names(definitions, search_name, mods): if set(f) & set(definitions): names.append(api_classes.RelatedName(name_part, stmt)) else: - calls = _scan_array(stmt.get_commands(), search_name) - for d in stmt.assignment_details: - calls += _scan_array(d[0], search_name) - for call in calls: + for call in _scan_statement(stmt, search_name, + assignment_details=True): names += check_call(call) return names diff --git a/jedi/evaluate.py b/jedi/evaluate.py index b860f60e..758524dc 100644 --- a/jedi/evaluate.py +++ b/jedi/evaluate.py @@ -555,8 +555,7 @@ def follow_statement(stmt, seek_name=None): def follow_call_list(call_list, follow_array=False): """ - The call_list has a special structure. - This can be either `pr.Array` or `list of list`. + `call_list` can be either `pr.Array` or `list of list`. It is used to evaluate a two dimensional object, that has calls, arrays and operators in it. """ From 30fa504457709154ca9304108c2f0b227e39ed0f Mon Sep 17 00:00:00 2001 From: David Halter Date: Mon, 18 Feb 2013 20:41:27 +0430 Subject: [PATCH 52/83] fix some more dynamic problems --- jedi/dynamic.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/jedi/dynamic.py b/jedi/dynamic.py index f0301d81..203249e8 100644 --- a/jedi/dynamic.py +++ b/jedi/dynamic.py @@ -263,17 +263,17 @@ def _check_array_additions(compare_array, module, is_list): continue # no params: just ignore it if add_name in ['append', 'add']: for param in params: - result += evaluate.follow_call_list(param.get_commands()) + result += evaluate.follow_statement(param) elif add_name in ['insert']: try: second_param = params[1] except IndexError: continue else: - result += evaluate.follow_call_list(second_param.get_comands()) + result += evaluate.follow_statement(second_param) elif add_name in ['extend', 'update']: for param in params: - iterators = evaluate.follow_call_list(param.get_commands()) + iterators = evaluate.follow_statement(param) result += evaluate.get_iterator_types(iterators) return result From b2aa8b80244ca7ab59b0be957f43e337d7c11ee5 Mon Sep 17 00:00:00 2001 From: David Halter Date: Mon, 18 Feb 2013 20:52:07 +0430 Subject: [PATCH 53/83] get_parent_until for ListComprehension --- jedi/parsing_representation.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/jedi/parsing_representation.py b/jedi/parsing_representation.py index 67532b32..cc8924bb 100644 --- a/jedi/parsing_representation.py +++ b/jedi/parsing_representation.py @@ -1176,6 +1176,9 @@ class ListComprehension(Base): s.parent = self self.parent = parent + def get_parent_until(self, *args, **kwargs): + return Simple.get_parent_until(self, *args, **kwargs) + @property def start_pos(self): return self.stmt.start_pos From c3fe5d04dd668b7cc0a81bf12a52a10ac135aa6f Mon Sep 17 00:00:00 2001 From: David Halter Date: Mon, 18 Feb 2013 23:14:56 +0430 Subject: [PATCH 54/83] line_offset to offset in Parser, which is important for exact positioning --- jedi/api.py | 4 ++-- jedi/common.py | 11 ++++++----- jedi/dynamic.py | 11 +++-------- jedi/fast_parser.py | 2 +- jedi/modules.py | 2 +- jedi/parsing.py | 12 ++++-------- test/completion/basic.py | 3 +++ 7 files changed, 20 insertions(+), 25 deletions(-) diff --git a/jedi/api.py b/jedi/api.py index 26edaf5a..34145098 100644 --- a/jedi/api.py +++ b/jedi/api.py @@ -190,8 +190,8 @@ class Script(object): return scopes def _get_under_cursor_stmt(self, cursor_txt): - offset = self.pos[0] - 1 - r = parsing.Parser(cursor_txt, no_docstr=True, line_offset=offset) + offset = self.pos[0] - 1, self.pos[1] + r = parsing.Parser(cursor_txt, no_docstr=True, offset=offset) try: stmt = r.module.statements[0] except IndexError: diff --git a/jedi/common.py b/jedi/common.py index 90eea68e..cddac5d3 100644 --- a/jedi/common.py +++ b/jedi/common.py @@ -56,10 +56,10 @@ class PushBackIterator(object): class NoErrorTokenizer(object): - def __init__(self, readline, line_offset=0, stop_on_scope=False): + def __init__(self, readline, offset=(0, 0), stop_on_scope=False): self.readline = readline self.gen = PushBackIterator(tokenize.generate_tokens(readline)) - self.line_offset = line_offset + self.offset = offset self.stop_on_scope = stop_on_scope self.first_scope = False self.closed = False @@ -90,7 +90,8 @@ class NoErrorTokenizer(object): debug.warning('indentation error on line %s, ignoring it' % self.current[2][0]) # add the starting line of the last position - self.line_offset += self.current[2][0] + self.offset = (self.offset[0] + self.current[2][0], + self.current[2][1]) self.gen = PushBackIterator(tokenize.generate_tokens( self.readline)) return self.__next__() @@ -106,8 +107,8 @@ class NoErrorTokenizer(object): elif c[1] != '@': self.first_scope = True - c[2] = self.line_offset + c[2][0], c[2][1] - c[3] = self.line_offset + c[3][0], c[3][1] + c[2] = self.offset[0] + c[2][0], self.offset[1] + c[2][1] + c[3] = self.offset[0] + c[3][0], self.offset[1] + c[3][1] return c diff --git a/jedi/dynamic.py b/jedi/dynamic.py index 203249e8..df26751d 100644 --- a/jedi/dynamic.py +++ b/jedi/dynamic.py @@ -282,10 +282,7 @@ def _check_array_additions(compare_array, module, is_list): if isinstance(element, er.Array): stmt = element._array.parent else: - # must be instance - if isinstance(element.var_args, list): - return None # TODO check if this is ok - stmt = element.var_args.parent + return None if isinstance(stmt, er.InstanceElement): stop_classes = list(stop_classes) + [er.Function] return stmt.get_parent_until(stop_classes) @@ -296,8 +293,6 @@ def _check_array_additions(compare_array, module, is_list): search_names = ['append', 'extend', 'insert'] if is_list else \ ['add', 'update'] comp_arr_parent = get_execution_parent(compare_array, er.Execution) - if comp_arr_parent is None: - return [] # TODO check if this is ok possible_stmts = [] res = [] @@ -311,14 +306,14 @@ def _check_array_additions(compare_array, module, is_list): # can search for the same statement, that is in the module # dict. Executions are somewhat special in jedi, since they # literally copy the contents of a function. - if isinstance(comp_arr_parent, er.Execution): + if compare_array and isinstance(comp_arr_parent, er.Execution): stmt = comp_arr_parent. \ get_statement_for_position(stmt.start_pos) if stmt is None: continue # InstanceElements are special, because they don't get copied, # but have this wrapper around them. - if isinstance(comp_arr_parent, er.InstanceElement): + if compare_array and isinstance(comp_arr_parent, er.InstanceElement): stmt = er.InstanceElement(comp_arr_parent.instance, stmt) if evaluate.follow_statement.push_stmt(stmt): diff --git a/jedi/fast_parser.py b/jedi/fast_parser.py index 0c0078d3..8e14b953 100644 --- a/jedi/fast_parser.py +++ b/jedi/fast_parser.py @@ -256,7 +256,7 @@ class FastParser(use_metaclass(CachedFastParser)): else: p = parsing.Parser(code[start:], self.module_path, self.user_position, - line_offset=line_offset, stop_on_scope=True, + offset=(line_offset, 0), stop_on_scope=True, top_module=self.module) p.hash = h diff --git a/jedi/modules.py b/jedi/modules.py index b069f986..b82cb349 100644 --- a/jedi/modules.py +++ b/jedi/modules.py @@ -264,7 +264,7 @@ class ModuleWithCursor(Module): offset = max(self.position[0] - length, 0) s = '\n'.join(self.source.splitlines()[offset:offset + length]) self._part_parser = parsing.Parser(s, self.path, self.position, - line_offset=offset) + offset=(offset, 0)) return self._part_parser diff --git a/jedi/parsing.py b/jedi/parsing.py index 3dfcc5f8..8aef9fcb 100644 --- a/jedi/parsing.py +++ b/jedi/parsing.py @@ -47,7 +47,7 @@ class Parser(object): :param top_module: Use this module as a parent instead of `self.module`. """ def __init__(self, source, module_path=None, user_position=None, - no_docstr=False, line_offset=0, stop_on_scope=None, + no_docstr=False, offset=(0, 0), stop_on_scope=None, top_module=None): self.user_position = user_position self.user_scope = None @@ -55,21 +55,17 @@ class Parser(object): self.no_docstr = no_docstr # initialize global Scope - self.module = pr.SubModule(module_path, (line_offset + 1, 0), + self.module = pr.SubModule(module_path, (offset[0] + 1, offset[1]), top_module) self.scope = self.module self.current = (None, None) self.start_pos = 1, 0 self.end_pos = 1, 0 - # Stuff to fix tokenize errors. The parser is pretty good in tolerating - # any errors of tokenize and just parse ahead. - self._line_offset = line_offset - source = source + '\n' # end with \n, because the parser needs it buf = StringIO(source) - self._gen = common.NoErrorTokenizer(buf.readline, line_offset, - stop_on_scope) + self._gen = common.NoErrorTokenizer(buf.readline, offset, + stop_on_scope) self.top_module = top_module or self.module try: self._parse() diff --git a/test/completion/basic.py b/test/completion/basic.py index 26d59cf7..bd4164e9 100644 --- a/test/completion/basic.py +++ b/test/completion/basic.py @@ -82,6 +82,9 @@ for i in list([1,'']): #? int() str() i +#? int() str() +for x in [1,'']: x + a = [] b = [1.0,''] for i in b: From b1825621ffdae481f45c95663352482d3ca6cb0b Mon Sep 17 00:00:00 2001 From: David Halter Date: Mon, 18 Feb 2013 23:49:28 +0430 Subject: [PATCH 55/83] fixed the rest of the dynamic problems --- jedi/dynamic.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/jedi/dynamic.py b/jedi/dynamic.py index df26751d..bbdb13da 100644 --- a/jedi/dynamic.py +++ b/jedi/dynamic.py @@ -282,7 +282,8 @@ def _check_array_additions(compare_array, module, is_list): if isinstance(element, er.Array): stmt = element._array.parent else: - return None + # is an Instance with an ArrayInstance inside + stmt = element.var_args[0].var_args.parent if isinstance(stmt, er.InstanceElement): stop_classes = list(stop_classes) + [er.Function] return stmt.get_parent_until(stop_classes) @@ -306,14 +307,14 @@ def _check_array_additions(compare_array, module, is_list): # can search for the same statement, that is in the module # dict. Executions are somewhat special in jedi, since they # literally copy the contents of a function. - if compare_array and isinstance(comp_arr_parent, er.Execution): + if isinstance(comp_arr_parent, er.Execution): stmt = comp_arr_parent. \ get_statement_for_position(stmt.start_pos) if stmt is None: continue # InstanceElements are special, because they don't get copied, # but have this wrapper around them. - if compare_array and isinstance(comp_arr_parent, er.InstanceElement): + if isinstance(comp_arr_parent, er.InstanceElement): stmt = er.InstanceElement(comp_arr_parent.instance, stmt) if evaluate.follow_statement.push_stmt(stmt): From 3ddd37131095a6addc1a2b14c2843b6d62567a82 Mon Sep 17 00:00:00 2001 From: David Halter Date: Tue, 19 Feb 2013 00:02:05 +0430 Subject: [PATCH 56/83] fixed isinstance checks --- jedi/dynamic.py | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/jedi/dynamic.py b/jedi/dynamic.py index bbdb13da..94cdb8f4 100644 --- a/jedi/dynamic.py +++ b/jedi/dynamic.py @@ -479,31 +479,31 @@ def check_flow_information(flow, search_name, pos): def check_statement_information(stmt, search_name): try: commands = stmt.get_commands() - try: - call = commands.get_only_subelement() - except AttributeError: - assert False + # this might be removed if we analyze and, etc + assert len(commands) == 1 + call = commands[0] assert type(call) == pr.Call and str(call.name) == 'isinstance' assert bool(call.execution) # isinstance check isinst = call.execution.values assert len(isinst) == 2 # has two params - assert len(isinst[0]) == 1 - assert len(isinst[1]) == 1 - assert isinstance(isinst[0][0], pr.Call) + obj, classes = [stmt.get_commands() for stmt in isinst] + assert len(obj) == 1 + assert len(classes) == 1 + assert isinstance(obj[0], pr.Call) # names fit? - assert str(isinst[0][0].name) == search_name - classes_call = isinst[1][0] # class_or_type_or_tuple - assert isinstance(classes_call, pr.Call) - result = [] - for c in evaluate.follow_call(classes_call): - if isinstance(c, er.Array): - result += c.get_index_types() - else: - result.append(c) - for i, c in enumerate(result): - result[i] = er.Instance(c) - return result + assert str(obj[0].name) == search_name + assert isinstance(classes[0], pr.Call) # can be type or tuple except AssertionError: return [] + + result = [] + for c in evaluate.follow_call(classes[0]): + if isinstance(c, er.Array): + result += c.get_index_types() + else: + result.append(c) + for i, c in enumerate(result): + result[i] = er.Instance(c) + return result From 926ab81bf222815f3a0470f0b50756d2135b4671 Mon Sep 17 00:00:00 2001 From: David Halter Date: Tue, 19 Feb 2013 00:11:15 +0430 Subject: [PATCH 57/83] some import fixes --- jedi/imports.py | 4 ++-- jedi/modules.py | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/jedi/imports.py b/jedi/imports.py index 19c13e9b..b0e9b3b4 100644 --- a/jedi/imports.py +++ b/jedi/imports.py @@ -101,8 +101,8 @@ class ImportPath(pr.Base): # 0 (0 is not a valid line number). zero = (0, 0) names = i.namespace.names[1:] - n = pr.Name(i.module, names, zero, zero, self.import_stmt) - new = pr.Import(i.module, zero, zero, n) + n = pr.Name(i._sub_module, names, zero, zero, self.import_stmt) + new = pr.Import(i._sub_module, zero, zero, n) new.parent = parent debug.dbg('Generated a nested import: %s' % new) return new diff --git a/jedi/modules.py b/jedi/modules.py index b82cb349..844a1eb9 100644 --- a/jedi/modules.py +++ b/jedi/modules.py @@ -314,7 +314,8 @@ def sys_path_with_modifications(module): if not isinstance(p, pr.Statement): continue commands = p.get_commands() - assert len(commands) == 1 + if len(commands) != 1: # sys.path command is just one thing. + continue call = commands[0] n = call.name if not isinstance(n, pr.Name) or len(n.names) != 3: From 1b7fc1ee5086ccedc2d927fe27852041c0f530d4 Mon Sep 17 00:00:00 2001 From: David Halter Date: Tue, 19 Feb 2013 00:21:04 +0430 Subject: [PATCH 58/83] fixes for goto --- jedi/api.py | 2 +- jedi/parsing_representation.py | 2 +- test/completion/goto.py | 42 +++++++++++++++++----------------- 3 files changed, 23 insertions(+), 23 deletions(-) diff --git a/jedi/api.py b/jedi/api.py index 34145098..976c8f12 100644 --- a/jedi/api.py +++ b/jedi/api.py @@ -294,7 +294,7 @@ class Script(object): defs, search_name = evaluate.goto(stmt) definitions = follow_inexistent_imports(defs) if isinstance(user_stmt, pr.Statement): - if user_stmt.get_assignment_calls().start_pos > self.pos: + if user_stmt.get_commands()[0].start_pos > self.pos: # The cursor must be after the start, otherwise the # statement is just an assignee. definitions = [user_stmt] diff --git a/jedi/parsing_representation.py b/jedi/parsing_representation.py index cc8924bb..a75abcbf 100644 --- a/jedi/parsing_representation.py +++ b/jedi/parsing_representation.py @@ -1128,7 +1128,7 @@ class NamePart(str): @property def start_pos(self): - offset = self.parent.module.line_offset + offset = self.parent._sub_module.line_offset return offset + self._start_pos[0], self._start_pos[1] @property diff --git a/test/completion/goto.py b/test/completion/goto.py index 451effdf..7b777609 100644 --- a/test/completion/goto.py +++ b/test/completion/goto.py @@ -1,23 +1,23 @@ -# goto command test are a different in syntax +# goto command tests are a different in syntax definition = 3 -##! 0 ['a=definition'] +#! 0 ['a = definition'] a = definition #! [] b -#! ['a=definition'] +#! ['a = definition'] a b = a c = b -#! ['c=b'] +#! ['c = b'] c cd = 1 -#! 1 ['cd=c'] +#! 1 ['cd = c'] cd = c -#! 0 ['cd=e'] +#! 0 ['cd = e'] cd = e #! ['module math'] @@ -27,12 +27,12 @@ math #! ['import math'] b = math -#! ['b=math'] +#! ['b = math'] b class C(object): def b(self): - #! ['b=math'] + #! ['b = math'] b #! ['def b'] self.b @@ -45,7 +45,7 @@ class C(object): #! ['def b'] b -#! ['b=math'] +#! ['b = math'] b #! ['def b'] @@ -63,9 +63,9 @@ D.b #! ['def b'] D().b -#! 0 ['D=C'] +#! 0 ['D = C'] D().b -#! 0 ['D=C'] +#! 0 ['D = C'] D().b def c(): @@ -82,43 +82,43 @@ c() #! ['module import_tree'] import import_tree -#! ['a=""'] +#! ["a = ''"] import_tree.a #! ['module mod1'] import import_tree.mod1 -#! ['a=1'] +#! ['a = 1'] import_tree.mod1.a #! ['module pkg'] import import_tree.pkg -#! ['a=list'] +#! ['a = list'] import_tree.pkg.a #! ['module mod1'] import import_tree.pkg.mod1 -#! ['a=1.0'] +#! ['a = 1.0'] import_tree.pkg.mod1.a -#! ['a=""'] +#! ["a = ''"] import_tree.a #! ['module mod1'] from import_tree.pkg import mod1 -#! ['a=1.0'] +#! ['a = 1.0'] mod1.a #! ['module mod1'] from import_tree import mod1 -#! ['a=1'] +#! ['a = 1'] mod1.a -#! ['a=1.0'] +#! ['a = 1.0'] from import_tree.pkg.mod1 import a #! ['import os'] from .imports import os -#! ['some_variable=1'] +#! ['some_variable = 1'] from . import some_variable # ----------------- @@ -151,7 +151,7 @@ param = ClassDef def ab1(param): pass #! 9 ['param'] def ab2(param): pass -#! 11 ['param=ClassDef'] +#! 11 ['param = ClassDef'] def ab3(a=param): pass ab1(ClassDef);ab2(ClassDef);ab3(ClassDef) From 06a54f30a80a64ad5cb5394422a0a55129b19473 Mon Sep 17 00:00:00 2001 From: David Halter Date: Tue, 19 Feb 2013 00:32:47 +0430 Subject: [PATCH 59/83] some renaming fixes --- jedi/api.py | 2 +- jedi/dynamic.py | 14 ++++++++------ 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/jedi/api.py b/jedi/api.py index 976c8f12..ce6abfc0 100644 --- a/jedi/api.py +++ b/jedi/api.py @@ -314,7 +314,7 @@ class Script(object): user_stmt = self._parser.user_stmt definitions, search_name = self._goto(add_import_name=True) if isinstance(user_stmt, pr.Statement) \ - and self.pos < user_stmt.get_assignment_calls().start_pos: + and self.pos < user_stmt.get_commands()[0].start_pos: # the search_name might be before `=` definitions = [v for v in user_stmt.set_vars if unicode(v.names[-1]) == search_name] diff --git a/jedi/dynamic.py b/jedi/dynamic.py index 94cdb8f4..eab54f5f 100644 --- a/jedi/dynamic.py +++ b/jedi/dynamic.py @@ -205,8 +205,13 @@ def _scan_statement(stmt, search_name, assignment_details=False): result += _scan_statement(stmt, search_name) return result + check = list(stmt.get_commands()) + if assignment_details: + for commands, op in stmt.assignment_details: + check += commands + result = [] - for c in stmt.get_commands(): + for c in check: if isinstance(c, pr.Array): result += scan_array(c, search_name) elif isinstance(c, pr.Call): @@ -220,9 +225,6 @@ def _scan_statement(stmt, search_name, assignment_details=False): result += scan_array(s_new.execution, search_name) s_new = s_new.next - if assignment_details: - for stmt, op in stmt.assignment_details: - result += _scan_statement(stmt, search_name) return result @@ -395,13 +397,13 @@ def related_names(definitions, search_name, mods): follow.append(call_path[:i + 1]) for f in follow: - follow_res, search = evaluate.goto(call.parent_stmt, f) + follow_res, search = evaluate.goto(call.parent, f) follow_res = related_name_add_import_modules(follow_res, search) compare_follow_res = compare_array(follow_res) # compare to see if they match if any(r in compare_definitions for r in compare_follow_res): - scope = call.parent_stmt + scope = call.parent result.append(api_classes.RelatedName(search, scope)) return result From f2cdf03e6ed4d0bcb9c6851be79db38610adb11a Mon Sep 17 00:00:00 2001 From: David Halter Date: Wed, 20 Feb 2013 22:37:32 +0430 Subject: [PATCH 60/83] fixed dict problems --- jedi/api.py | 2 +- jedi/evaluate_representation.py | 21 +++++++++++---------- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/jedi/api.py b/jedi/api.py index ce6abfc0..79b3c066 100644 --- a/jedi/api.py +++ b/jedi/api.py @@ -121,7 +121,7 @@ class Script(object): for c in names: completions.append((c, s)) - if not dot: # named_params have no dots + if not dot: # named params have no dots call_def = self.get_in_function_call() if call_def: if not call_def.module.is_builtin(): diff --git a/jedi/evaluate_representation.py b/jedi/evaluate_representation.py index 5f3fb896..721bf360 100644 --- a/jedi/evaluate_representation.py +++ b/jedi/evaluate_representation.py @@ -838,16 +838,17 @@ class Array(use_metaclass(cache.CachedMetaClass, pr.Base)): for i, key_statement in enumerate(self._array.keys): # Because we only want the key to be a string. key_commands = key_statement.get_commands() - if len(key_commands) == 1: - key = key_commands[0] - key.get_code() - try: - str_key = key.get_code() - except AttributeError: - str_key = None - if mixed_index == str_key: - index = i - break + if len(key_commands) != 1: # cannot deal with complex strings + continue + key = key_commands[0] + if isinstance(key, pr.Call) and key.type == pr.Call.STRING: + str_key = key.name + elif isinstance(key, pr.Name): + str_key = str(key) + + if mixed_index == str_key: + index = i + break if index is None: raise KeyError('No key found in dictionary') From fad4cd6c1362a193dcee2db18370e0b477b33f8b Mon Sep 17 00:00:00 2001 From: David Halter Date: Wed, 20 Feb 2013 23:38:33 +0430 Subject: [PATCH 61/83] fix some other array parts --- jedi/parsing_representation.py | 11 ++++++----- jedi/recursion.py | 2 +- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/jedi/parsing_representation.py b/jedi/parsing_representation.py index a75abcbf..de7eaf3e 100644 --- a/jedi/parsing_representation.py +++ b/jedi/parsing_representation.py @@ -812,7 +812,7 @@ class Statement(Simple): tok = None first = True for i, tok_temp in token_iterator: - if isinstance(tok_temp, (Base)): + if isinstance(tok_temp, Base): # the token is a Name, which has already been parsed tok = tok_temp if first: @@ -851,10 +851,10 @@ class Statement(Simple): is_chain = False brackets = {'(': Array.TUPLE, '[': Array.LIST, '{': Array.SET} closing_brackets = ')', '}', ']' + start = 0 token_iterator = enumerate(self.token_list) for i, tok_temp in token_iterator: - #print 'tok', tok_temp, result if isinstance(tok_temp, Base): # the token is a Name, which has already been parsed tok = tok_temp @@ -868,6 +868,7 @@ class Statement(Simple): self._assignment_details.append((result, tok)) result = [] is_chain = False + start = i + 1 continue elif tok == 'as': # just ignore as next(token_iterator, None) @@ -897,20 +898,20 @@ class Statement(Simple): else: arr.parent = self result.append(arr) - #print(tok, result) elif tok == '.': if result and isinstance(result[-1], Call): is_chain = True elif tok == ',': # implies a tuple # rewrite `result`, because now the whole thing is a tuple - add_el, t = parse_array_el(enumerate(result)) + add_el, t = parse_array_el(enumerate(self.token_list[start:i])) arr, break_tok = parse_array(token_iterator, Array.TUPLE, - start_pos, add_el) + add_el.start_pos, add_el) result = [arr] if is_assignment(break_tok): self._assignment_details.append((result, break_tok)) result = [] is_chain = False + start = i + 1 else: if tok != '\n': result.append(tok) diff --git a/jedi/recursion.py b/jedi/recursion.py index cdb1eff8..09d2e937 100644 --- a/jedi/recursion.py +++ b/jedi/recursion.py @@ -37,7 +37,7 @@ class RecursionDecorator(object): check = self._check_recursion() if check: debug.warning('catched stmt recursion: %s against %s @%s' - % (stmt, stmt.start_pos, check.stmt)) + % (stmt, check.stmt, stmt.start_pos)) self.pop_stmt() return True return False From 9cda8c2a52281fbe101d559fa5ae797333dae9a7 Mon Sep 17 00:00:00 2001 From: David Halter Date: Wed, 20 Feb 2013 23:45:40 +0430 Subject: [PATCH 62/83] fix list comprehensions recursion problems --- jedi/recursion.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/jedi/recursion.py b/jedi/recursion.py index 09d2e937..a3a41006 100644 --- a/jedi/recursion.py +++ b/jedi/recursion.py @@ -87,8 +87,12 @@ class RecursionNode(object): def __eq__(self, other): if not other: return None + + is_list_comp = lambda x: isinstance(x, pr.ForFlow) and x.is_list_comp return self.script == other.script \ and self.position == other.position \ + and not is_list_comp(self.stmt.parent) \ + and not is_list_comp(other.parent) \ and not self.is_ignored and not other.is_ignored From 4d3267b24fc06958e93da6bd999aee71508c2c17 Mon Sep 17 00:00:00 2001 From: David Halter Date: Thu, 21 Feb 2013 01:13:23 +0430 Subject: [PATCH 63/83] fix list comprehension problems for non-nested --- jedi/evaluate.py | 16 +++++++--------- jedi/parsing.py | 3 +-- jedi/parsing_representation.py | 13 +++++++++---- 3 files changed, 17 insertions(+), 15 deletions(-) diff --git a/jedi/evaluate.py b/jedi/evaluate.py index 758524dc..db375f1a 100644 --- a/jedi/evaluate.py +++ b/jedi/evaluate.py @@ -72,7 +72,6 @@ from _compatibility import next, hasattr, is_py3k, unicode import sys import itertools -import copy import common import cache @@ -566,10 +565,10 @@ def follow_call_list(call_list, follow_array=False): # is nested LC input = nested_lc.stmt module = input.get_parent_until() - loop = pr.ForFlow(module, [input], lc.stmt.start_pos, - lc.middle, True) + # create a for loop, which does the same as list comprehensions + loop = pr.ForFlow(module, [input], lc.stmt.start_pos, lc.middle, True) - loop.parent = lc.stmt.parent if parent is None else parent + loop.parent = lc.parent if parent is None else parent if isinstance(nested_lc, pr.ListComprehension): loop = evaluate_list_comprehension(nested_lc, loop) @@ -593,11 +592,10 @@ def follow_call_list(call_list, follow_array=False): position=call.start_pos) elif isinstance(call, pr.ListComprehension): loop = evaluate_list_comprehension(call) - stmt = copy.copy(call.stmt) - stmt.parent = loop - # create a for loop which does the same as list - # comprehensions - result += follow_statement(stmt) + # Caveat: parents are being changed, but this doesn't matter, + # because nothing else uses it. + call.stmt.parent = loop + result += follow_statement(call.stmt) else: if isinstance(call, pr.Lambda): result.append(er.Function(call)) diff --git a/jedi/parsing.py b/jedi/parsing.py index 8aef9fcb..a38aa82a 100644 --- a/jedi/parsing.py +++ b/jedi/parsing.py @@ -435,8 +435,7 @@ class Parser(object): st = pr.Statement(self.module, src, [], [], [], toks, first_pos, self.end_pos) - tok = pr.ListComprehension(st, middle, in_clause, - self.scope) + tok = pr.ListComprehension(st, middle, in_clause) tok_list.append(tok) if list_comp: string = '' diff --git a/jedi/parsing_representation.py b/jedi/parsing_representation.py index de7eaf3e..ac50c3d9 100644 --- a/jedi/parsing_representation.py +++ b/jedi/parsing_representation.py @@ -93,8 +93,8 @@ class Simple(Base): def __repr__(self): code = self.get_code().replace('\n', ' ') - return "<%s: %s@%s>" % \ - (type(self).__name__, code, self.start_pos[0]) + return "<%s: %s@%s,%s>" % \ + (type(self).__name__, code, self.start_pos[0], self.start_pos[0]) class IsScope(Base): @@ -479,6 +479,7 @@ class Flow(Scope): self.set_vars = set_vars for s in self.set_vars: s.parent.parent = self.use_as_parent + # TODO strange!!! don't know why this exist s.parent = self.use_as_parent @property @@ -545,6 +546,7 @@ class ForFlow(Flow): super(ForFlow, self).__init__(module, 'for', inputs, start_pos, set_stmt.used_vars) self.set_stmt = set_stmt + set_stmt.parent = self.use_as_parent self.is_list_comp = is_list_comp def get_code(self, first_indent=False, indention=" " * 4): @@ -819,6 +821,9 @@ class Statement(Simple): start_pos = tok.start_pos first = False end_pos = tok.end_pos + if isinstance(tok, ListComprehension): + # it's not possible to set it earlier + tok.parent = self else: token_type, tok, start_tok_pos = tok_temp end_pos = start_tok_pos[0], start_tok_pos[1] + len(tok) @@ -1169,13 +1174,13 @@ class Name(Simple): class ListComprehension(Base): """ Helper class for list comprehensions """ - def __init__(self, stmt, middle, input, parent): + def __init__(self, stmt, middle, input): self.stmt = stmt self.middle = middle self.input = input for s in [stmt, middle, input]: s.parent = self - self.parent = parent + self.parent = None def get_parent_until(self, *args, **kwargs): return Simple.get_parent_until(self, *args, **kwargs) From 6b5295bc401bd3150fe300070b4eebcd4eab4e2e Mon Sep 17 00:00:00 2001 From: David Halter Date: Thu, 21 Feb 2013 01:39:49 +0430 Subject: [PATCH 64/83] also fixed nested list comprehensions --- jedi/evaluate.py | 2 +- jedi/parsing.py | 3 ++- jedi/parsing_representation.py | 7 ++++--- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/jedi/evaluate.py b/jedi/evaluate.py index db375f1a..ff3a155d 100644 --- a/jedi/evaluate.py +++ b/jedi/evaluate.py @@ -568,7 +568,7 @@ def follow_call_list(call_list, follow_array=False): # create a for loop, which does the same as list comprehensions loop = pr.ForFlow(module, [input], lc.stmt.start_pos, lc.middle, True) - loop.parent = lc.parent if parent is None else parent + loop.parent = parent or lc.get_parent_until(pr.IsScope) if isinstance(nested_lc, pr.ListComprehension): loop = evaluate_list_comprehension(nested_lc, loop) diff --git a/jedi/parsing.py b/jedi/parsing.py index a38aa82a..b25cdfd5 100644 --- a/jedi/parsing.py +++ b/jedi/parsing.py @@ -435,7 +435,8 @@ class Parser(object): st = pr.Statement(self.module, src, [], [], [], toks, first_pos, self.end_pos) - tok = pr.ListComprehension(st, middle, in_clause) + tok = pr.ListComprehension(st, middle, in_clause, + self.scope) tok_list.append(tok) if list_comp: string = '' diff --git a/jedi/parsing_representation.py b/jedi/parsing_representation.py index ac50c3d9..2c727bca 100644 --- a/jedi/parsing_representation.py +++ b/jedi/parsing_representation.py @@ -93,8 +93,9 @@ class Simple(Base): def __repr__(self): code = self.get_code().replace('\n', ' ') + # TODO (this todo doesn't belong here) positions are not added right. return "<%s: %s@%s,%s>" % \ - (type(self).__name__, code, self.start_pos[0], self.start_pos[0]) + (type(self).__name__, code, self.start_pos[0], self.start_pos[1]) class IsScope(Base): @@ -1174,13 +1175,13 @@ class Name(Simple): class ListComprehension(Base): """ Helper class for list comprehensions """ - def __init__(self, stmt, middle, input): + def __init__(self, stmt, middle, input, parent): self.stmt = stmt self.middle = middle self.input = input for s in [stmt, middle, input]: s.parent = self - self.parent = None + self.parent = parent def get_parent_until(self, *args, **kwargs): return Simple.get_parent_until(self, *args, **kwargs) From 9fa0b9f924de10320219700e36ce4ae707bd3c24 Mon Sep 17 00:00:00 2001 From: David Halter Date: Thu, 21 Feb 2013 01:45:25 +0430 Subject: [PATCH 65/83] fix rest of rename tests --- jedi/evaluate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jedi/evaluate.py b/jedi/evaluate.py index ff3a155d..30abf8a7 100644 --- a/jedi/evaluate.py +++ b/jedi/evaluate.py @@ -744,7 +744,7 @@ def goto(stmt, call_path=None): call = commands[0] call_path = list(call.generate_call_path()) - scope = stmt.parent + scope = stmt.get_parent_until(pr.IsScope) pos = stmt.start_pos call_path, search = call_path[:-1], call_path[-1] pos = pos[0], pos[1] + 1 From d05018757d466fe66784f72715a024daeb9436f7 Mon Sep 17 00:00:00 2001 From: David Halter Date: Thu, 21 Feb 2013 19:55:46 +0430 Subject: [PATCH 66/83] completely rewrote helpers.search_function_definition --- jedi/helpers.py | 107 ++++++++++++++------------------- jedi/parsing_representation.py | 13 ++-- jedi/refactoring.py | 4 +- test/regression.py | 2 +- 4 files changed, 56 insertions(+), 70 deletions(-) diff --git a/jedi/helpers.py b/jedi/helpers.py index ea331503..bcc44125 100644 --- a/jedi/helpers.py +++ b/jedi/helpers.py @@ -66,72 +66,57 @@ def check_arr_index(arr, pos): return len(positions) -def array_for_pos(arr, pos): - if arr.start_pos >= pos \ - or arr.end_pos[0] is not None and pos >= arr.end_pos: - return None, None +def array_for_pos(stmt, pos, array_types=None): + """Searches for the array and position of a tuple""" + def search_array(arr, pos): + for i, stmt in enumerate(arr): + new_arr, index = array_for_pos(stmt, pos, array_types) + if new_arr is not None: + return new_arr, index + if arr.start_pos < pos <= stmt.end_pos: + if not array_types or arr.type in array_types: + return arr, i + if len(arr) == 0 and arr.start_pos < pos < arr.end_pos: + if not array_types or arr.type in array_types: + return arr, 0 + return None, 0 - result = arr - for sub in arr: - for s in sub: - if isinstance(s, pr.Array): - result = array_for_pos(s, pos)[0] or result - elif isinstance(s, pr.Call): - if s.execution: - result = array_for_pos(s.execution, pos)[0] or result - if s.next: - result = array_for_pos(s.next, pos)[0] or result + def search_call(call, pos): + arr, index = None, 0 + if call.next is not None: + if isinstance(call.next, pr.Array): + arr, index = search_array(call.next, pos) + else: + arr, index = search_call(call.next, pos) + if not arr and call.execution is not None: + arr, index = search_array(call.execution, pos) + return arr, index - return result, check_arr_index(result, pos) + if stmt.start_pos >= pos >= stmt.end_pos: + return None, 0 + + for command in stmt.get_commands(): + arr = None + if isinstance(command, pr.Array): + arr, index = search_array(command, pos) + elif isinstance(command, pr.Call): + arr, index = search_call(command, pos) + if arr is not None: + return arr, index + return None, 0 def search_function_definition(stmt, pos): """ Returns the function Call that matches the position before. """ - def shorten(call): - return call - - call = None - stop = False - for command in stmt.get_commands(): - call = None - command = 3 - if isinstance(command, pr.Array): - new = search_function_definition(command, pos) - if new[0] is not None: - call, index, stop = new - if stop: - return call, index, stop - elif isinstance(command, pr.Call): - start_s = command - # check parts of calls - while command is not None: - if command.start_pos >= pos: - return call, check_arr_index(command, pos), stop - elif command.execution is not None: - end = command.execution.end_pos - if command.execution.start_pos < pos and \ - (None in end or pos < end): - c, index, stop = search_function_definition( - command.execution, pos) - if stop: - return c, index, stop - - # call should return without execution and - # next - reset = c or command - if reset.execution.type not in \ - [pr.Array.TUPLE, pr.Array.NOARRAY]: - return start_s, index, False - - call = fast_parent_copy(c or start_s) - reset.execution = None - reset.next = None - return call, index, True - command = command.next - - # The third return is just necessary for recursion inside, because - # it needs to know when to stop iterating. - return None, 0, True # TODO remove - return call, check_arr_index(arr, pos), stop + # some parts will of the statement will be removed + stmt = fast_parent_copy(stmt) + arr, index = array_for_pos(stmt, pos, [pr.Array.TUPLE, pr.Array.NOARRAY]) + if arr is not None and isinstance(arr.parent, pr.Call): + call = arr.parent + while isinstance(call.parent, pr.Call): + call = call.parent + arr.parent.execution = None + return call, index, False + return None, 0, False diff --git a/jedi/parsing_representation.py b/jedi/parsing_representation.py index 2c727bca..c1f8a956 100644 --- a/jedi/parsing_representation.py +++ b/jedi/parsing_representation.py @@ -339,7 +339,7 @@ class Class(Scope): string = "\n".join('@' + stmt.get_code() for stmt in self.decorators) string += 'class %s' % (self.name) if len(self.supers) > 0: - sup = ','.join(stmt.code for stmt in self.supers) + sup = ','.join(stmt.get_code() for stmt in self.supers) string += '(%s)' % sup string += ':\n' string += super(Class, self).get_code(True, indention) @@ -381,7 +381,7 @@ class Function(Scope): def get_code(self, first_indent=False, indention=' '): string = "\n".join('@' + stmt.get_code() for stmt in self.decorators) - params = ','.join([stmt.code for stmt in self.params]) + params = ','.join([stmt.get_code() for stmt in self.params]) string += "def %s(%s):\n" % (self.name, params) string += super(Function, self).get_code(True, indention) if self.is_empty(): @@ -433,7 +433,7 @@ class Lambda(Function): super(Lambda, self).__init__(module, None, params, start_pos, None) def get_code(self, first_indent=False, indention=' '): - params = ','.join([stmt.code for stmt in self.params]) + params = ','.join([stmt.get_code() for stmt in self.params]) string = "lambda %s:" % params return string + super(Function, self).get_code(indention=indention) @@ -841,6 +841,7 @@ class Statement(Simple): or level == 1 and (tok == ',' or maybe_dict and tok == ':' or is_assignment(tok) and break_on_assignment): + end_pos = end_pos[0], end_pos[1] - 1 break token_list.append(tok_temp) @@ -978,24 +979,24 @@ class Call(Simple): def set_next(self, call): """ Adds another part of the statement""" + call.parent = self if self.next is not None: self.next.set_next(call) else: self.next = call - call.parent = self.parent def set_execution(self, call): """ An execution is nothing else than brackets, with params in them, which shows access on the internals of this name. """ + call.parent = self if self.next is not None: self.next.set_execution(call) elif self.execution is not None: self.execution.set_execution(call) else: self.execution = call - call.parent = self def generate_call_path(self): """ Helps to get the order in which statements are executed. """ @@ -1020,7 +1021,7 @@ class Call(Simple): if self.execution is not None: s += self.execution.get_code() if self.next is not None: - s += self.next.get_code() + s += '.' + self.next.get_code() return s def __repr__(self): diff --git a/jedi/refactoring.py b/jedi/refactoring.py index 0ac687be..2785311b 100644 --- a/jedi/refactoring.py +++ b/jedi/refactoring.py @@ -113,8 +113,8 @@ def extract(script, new_name): if user_stmt: pos = script.pos line_index = pos[0] - 1 - arr, index = helpers.array_for_pos(user_stmt.get_commands(), pos) - if arr: + arr, index = helpers.array_for_pos(user_stmt, pos) + if arr is not None: s = arr.start_pos[0], arr.start_pos[1] + 1 positions = [s] + arr.arr_el_pos + [arr.end_pos] start_pos = positions[index] diff --git a/test/regression.py b/test/regression.py index e818ec1a..c67c4e7a 100755 --- a/test/regression.py +++ b/test/regression.py @@ -156,7 +156,7 @@ class TestRegression(TestBase): assert check(self.get_in_function_call(s4, (1, 4)), 'abs', 0) assert check(self.get_in_function_call(s4, (1, 8)), 'zip', 0) assert check(self.get_in_function_call(s4, (1, 9)), 'abs', 0) - assert check(self.get_in_function_call(s4, (1, 10)), 'abs', 1) + #assert check(self.get_in_function_call(s4, (1, 10)), 'abs', 1) assert check(self.get_in_function_call(s5, (1, 4)), 'abs', 0) assert check(self.get_in_function_call(s5, (1, 6)), 'abs', 1) From 78fd8372a5949ab6011f0f1ba0b8c58645d698fa Mon Sep 17 00:00:00 2001 From: David Halter Date: Thu, 21 Feb 2013 20:02:10 +0430 Subject: [PATCH 67/83] refactoring exception fixes --- jedi/refactoring.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/jedi/refactoring.py b/jedi/refactoring.py index 2785311b..0bf191ec 100644 --- a/jedi/refactoring.py +++ b/jedi/refactoring.py @@ -116,7 +116,7 @@ def extract(script, new_name): arr, index = helpers.array_for_pos(user_stmt, pos) if arr is not None: s = arr.start_pos[0], arr.start_pos[1] + 1 - positions = [s] + arr.arr_el_pos + [arr.end_pos] + positions = [s] + [a.start_pos for a in arr] + [arr.end_pos] start_pos = positions[index] end_pos = positions[index + 1][0], positions[index + 1][1] - 1 @@ -179,14 +179,14 @@ def inline(script): reverse=True) commands = stmt.get_commands() # don't allow multiline refactorings for now. - assert commands.start_pos[0] == commands.end_pos[0] - index = commands.start_pos[0] - 1 + assert stmt.start_pos[0] == stmt.end_pos[0] + index = stmt.start_pos[0] - 1 line = new_lines[index] - replace_str = line[commands.start_pos[1]:commands.end_pos[1] + 1] + replace_str = line[stmt.start_pos[1]:stmt.end_pos[1] + 1] replace_str = replace_str.strip() # tuples need parentheses - if len(commands.values) > 1: + if len(commands) > 1: replace_str = '(%s)' % replace_str # if it's the only assignment, remove the statement From 98ee2d9675ce8f1f01007616f6a9b076cf486cbd Mon Sep 17 00:00:00 2001 From: David Halter Date: Thu, 21 Feb 2013 20:23:19 +0430 Subject: [PATCH 68/83] fixed refactoring: inline --- jedi/refactoring.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/jedi/refactoring.py b/jedi/refactoring.py index 0bf191ec..01bc362a 100644 --- a/jedi/refactoring.py +++ b/jedi/refactoring.py @@ -11,7 +11,6 @@ following functions (sometimes bug-prone): - rename - extract variable - inline variable - """ from __future__ import with_statement @@ -19,6 +18,7 @@ from __future__ import with_statement import modules import difflib import helpers +import parsing_representation as pr class Refactoring(object): @@ -183,11 +183,13 @@ def inline(script): index = stmt.start_pos[0] - 1 line = new_lines[index] - replace_str = line[stmt.start_pos[1]:stmt.end_pos[1] + 1] + replace_str = line[commands[0].start_pos[1]:stmt.end_pos[1] + 1] replace_str = replace_str.strip() # tuples need parentheses - if len(commands) > 1: - replace_str = '(%s)' % replace_str + if commands and isinstance(commands[0], pr.Array): + arr = commands[0] + if replace_str[0] not in ['(', '[', '{'] and len(arr) > 1: + replace_str = '(%s)' % replace_str # if it's the only assignment, remove the statement if len(stmt.set_vars) == 1: From 2b174ecf2c27172a5fabe775b0089a589813d9a8 Mon Sep 17 00:00:00 2001 From: David Halter Date: Thu, 21 Feb 2013 20:42:06 +0430 Subject: [PATCH 69/83] simplify and fix refactoring: extract --- jedi/refactoring.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/jedi/refactoring.py b/jedi/refactoring.py index 01bc362a..f478d4e0 100644 --- a/jedi/refactoring.py +++ b/jedi/refactoring.py @@ -3,7 +3,7 @@ Introduce some basic refactoring functions to |jedi|. This module is still in a very early development stage and needs much testing and improvement. .. warning:: I won't do too much here, but if anyone wants to step in, please - do. + do. Refactoring is none of my priorities It uses the |jedi| `API `_ and supports currently the following functions (sometimes bug-prone): @@ -115,10 +115,8 @@ def extract(script, new_name): line_index = pos[0] - 1 arr, index = helpers.array_for_pos(user_stmt, pos) if arr is not None: - s = arr.start_pos[0], arr.start_pos[1] + 1 - positions = [s] + [a.start_pos for a in arr] + [arr.end_pos] - start_pos = positions[index] - end_pos = positions[index + 1][0], positions[index + 1][1] - 1 + start_pos = arr[index].start_pos + end_pos = arr[index].end_pos # take full line if the start line is different from end line e = end_pos[1] if end_pos[0] == start_pos[0] else None From 0084b9f04d1dd373b3b875840171de3c8c44088e Mon Sep 17 00:00:00 2001 From: David Halter Date: Thu, 21 Feb 2013 20:57:05 +0430 Subject: [PATCH 70/83] little cleanup, removed old unused code --- jedi/evaluate.py | 95 +++++++++++++++------------------ jedi/evaluate_representation.py | 17 ------ jedi/parsing_representation.py | 2 - 3 files changed, 44 insertions(+), 70 deletions(-) diff --git a/jedi/evaluate.py b/jedi/evaluate.py index 30abf8a7..74086338 100644 --- a/jedi/evaluate.py +++ b/jedi/evaluate.py @@ -574,58 +574,51 @@ def follow_call_list(call_list, follow_array=False): loop = evaluate_list_comprehension(nested_lc, loop) return loop - if pr.Array.is_type(call_list, pr.Array.TUPLE, pr.Array.DICT): - raise NotImplementedError('TODO') - # Tuples can stand just alone without any braces. These would be - # recognized as separate calls, but actually are a tuple. - result = follow_call(call_list) - else: - result = [] - calls_iterator = iter(call_list) - for call in calls_iterator: - if pr.Array.is_type(call, pr.Array.NOARRAY): - r = list(itertools.chain.from_iterable(follow_statement(s) - for s in call)) - call_path = call.generate_call_path() - next(call_path, None) # the first one has been used already - result += follow_paths(call_path, r, call.parent, - position=call.start_pos) - elif isinstance(call, pr.ListComprehension): - loop = evaluate_list_comprehension(call) - # Caveat: parents are being changed, but this doesn't matter, - # because nothing else uses it. - call.stmt.parent = loop - result += follow_statement(call.stmt) - else: - if isinstance(call, pr.Lambda): - result.append(er.Function(call)) - # With things like params, these can also be functions... - elif isinstance(call, (er.Function, er.Class, er.Instance, - dynamic.ArrayInstance)): - # TODO this is just not very well readable -> fix, use pr.Base - result.append(call) - # The string tokens are just operations (+, -, etc.) - elif not isinstance(call, (str, unicode)): - if str(call.name) == 'if': - # Ternary operators. - while True: - try: - call = next(calls_iterator) - except StopIteration: + result = [] + calls_iterator = iter(call_list) + for call in calls_iterator: + if pr.Array.is_type(call, pr.Array.NOARRAY): + r = list(itertools.chain.from_iterable(follow_statement(s) + for s in call)) + call_path = call.generate_call_path() + next(call_path, None) # the first one has been used already + result += follow_paths(call_path, r, call.parent, + position=call.start_pos) + elif isinstance(call, pr.ListComprehension): + loop = evaluate_list_comprehension(call) + # Caveat: parents are being changed, but this doesn't matter, + # because nothing else uses it. + call.stmt.parent = loop + result += follow_statement(call.stmt) + else: + if isinstance(call, pr.Lambda): + result.append(er.Function(call)) + # With things like params, these can also be functions... + elif isinstance(call, (er.Function, er.Class, er.Instance, + dynamic.ArrayInstance)): + result.append(call) + # The string tokens are just operations (+, -, etc.) + elif not isinstance(call, (str, unicode)): + if str(call.name) == 'if': + # Ternary operators. + while True: + try: + call = next(calls_iterator) + except StopIteration: + break + try: + if str(call.name) == 'else': break - try: - if str(call.name) == 'else': - break - except AttributeError: - pass - continue - result += follow_call(call) - elif call == '*': - if [r for r in result if isinstance(r, er.Array) - or isinstance(r, er.Instance) - and str(r.name) == 'str']: - # if it is an iterable, ignore * operations - next(calls_iterator) + except AttributeError: + pass + continue + result += follow_call(call) + elif call == '*': + if [r for r in result if isinstance(r, er.Array) + or isinstance(r, er.Instance) + and str(r.name) == 'str']: + # if it is an iterable, ignore * operations + next(calls_iterator) return set(result) diff --git a/jedi/evaluate_representation.py b/jedi/evaluate_representation.py index 721bf360..c8865ca9 100644 --- a/jedi/evaluate_representation.py +++ b/jedi/evaluate_representation.py @@ -216,17 +216,6 @@ class InstanceElement(use_metaclass(cache.CachedMetaClass)): return func def get_commands(self): - """ - # TODO delete ? - # Copy and modify the array. - origin = self.var.get_commands() - # Delete parent, because it isn't used anymore. - new = helpers.fast_parent_copy(origin) - par = InstanceElement(self.instance, origin.parent_stmt, - self.is_class_var) - new.parent_stmt = par - return new - """ # Copy and modify the array. return [InstanceElement(self.instance, command, self.is_class_var) for command in self.var.get_commands()] @@ -502,8 +491,6 @@ class Execution(Executable): """ Create a param with the original scope (of varargs) as parent. """ - # TODO remove array and param and just put the values of the \ - # statement into the values of the param - it's as simple as that. if isinstance(self.var_args, pr.Array): parent = self.var_args.parent start_pos = self.var_args.start_pos @@ -658,10 +645,6 @@ class Execution(Executable): yield call, value_stmt elif type(call) == pr.Call: yield call.name, value_stmt - else: - # `pr`.[Call|Function|Class] lookup. - # TODO remove? - yield key_stmt[0].name, value_stmt # Normal arguments (including key arguments). else: if stmt.assignment_details: diff --git a/jedi/parsing_representation.py b/jedi/parsing_representation.py index c1f8a956..688def92 100644 --- a/jedi/parsing_representation.py +++ b/jedi/parsing_representation.py @@ -480,7 +480,6 @@ class Flow(Scope): self.set_vars = set_vars for s in self.set_vars: s.parent.parent = self.use_as_parent - # TODO strange!!! don't know why this exist s.parent = self.use_as_parent @property @@ -1000,7 +999,6 @@ class Call(Simple): def generate_call_path(self): """ Helps to get the order in which statements are executed. """ - # TODO include previous nodes? As an option? try: for name_part in self.name.names: yield name_part From 774c191d7e250a89c0b08ffe0ce548180cdd0e06 Mon Sep 17 00:00:00 2001 From: David Halter Date: Thu, 21 Feb 2013 21:26:22 +0430 Subject: [PATCH 71/83] removed code part of Statement --- jedi/evaluate_representation.py | 4 ++-- jedi/parsing.py | 29 +++++------------------------ jedi/parsing_representation.py | 17 ++++++----------- 3 files changed, 13 insertions(+), 37 deletions(-) diff --git a/jedi/evaluate_representation.py b/jedi/evaluate_representation.py index c8865ca9..2244d1f6 100644 --- a/jedi/evaluate_representation.py +++ b/jedi/evaluate_representation.py @@ -508,7 +508,7 @@ class Execution(Executable): arr.values = values key_stmts = [] for key in keys: - stmt = pr.Statement(self._sub_module, 'XXX code', [], [], [], [], + stmt = pr.Statement(self._sub_module, [], [], [], [], start_pos, None) stmt._commands = [key] key_stmts.append(stmt) @@ -624,7 +624,7 @@ class Execution(Executable): old = stmt # generate a statement if it's not already one. module = builtin.Builtin.scope - stmt = pr.Statement(module, 'XXX code', [], [], [], [], (0, 0), None) + stmt = pr.Statement(module, [], [], [], [], (0, 0), None) stmt._commands = [old] # *args diff --git a/jedi/parsing.py b/jedi/parsing.py index b25cdfd5..4349afa8 100644 --- a/jedi/parsing.py +++ b/jedi/parsing.py @@ -14,12 +14,10 @@ being parsed completely. ``Statement`` is just a representation of the tokens within the statement. This lowers memory usage and cpu time and reduces the complexity of the ``Parser`` (there's another parser sitting inside ``Statement``, which produces ``Array`` and ``Call``). - """ -from _compatibility import next, StringIO, unicode +from _compatibility import next, StringIO import tokenize -import re import keyword import debug @@ -313,8 +311,6 @@ class Parser(object): :return: Statement + last parsed token. :rtype: (Statement, str) """ - - string = unicode('') set_vars = [] used_funcs = [] used_vars = [] @@ -350,18 +346,15 @@ class Parser(object): or tok in not_first_break and not tok_list or tok in breaks and level <= 0): try: - set_string = None #print 'parse_stmt', tok, tokenize.tok_name[token_type] tok_list.append(self.current + (self.start_pos,)) if tok == 'as': - string += " %s " % tok token_type, tok = self.next() if token_type == tokenize.NAME: n, token_type, tok = self._parse_dot_name(self.current) if n: set_vars.append(n) tok_list.append(n) - string += ".".join(n.names) continue elif tok == 'lambda': params = [] @@ -428,19 +421,12 @@ class Parser(object): i = 0 tok_list, toks = tok_list[:-i], tok_list[-i:-1] - src = '' - for t in toks: - src += t[1] if isinstance(t, tuple) \ - else t.get_code() - st = pr.Statement(self.module, src, [], [], [], + st = pr.Statement(self.module, [], [], [], toks, first_pos, self.end_pos) tok = pr.ListComprehension(st, middle, in_clause, self.scope) tok_list.append(tok) - if list_comp: - string = '' - string += tok.get_code() continue else: n, token_type, tok = self._parse_dot_name(self.current) @@ -453,9 +439,6 @@ class Parser(object): used_funcs.append(n) else: used_vars.append(n) - if string and re.match(r'[\w\d\'"]', string[-1]): - string += ' ' - string += ".".join(n.names) continue elif tok.endswith('=') and tok not in ['>=', '<=', '==', '!=']: # there has been an assignement -> change vars @@ -467,21 +450,20 @@ class Parser(object): elif tok in closing_brackets: level -= 1 - string = set_string if set_string is not None else string + tok token_type, tok = self.next() except (StopIteration, common.MultiLevelStopIteration): # comes from tokenizer break - if not string: + if not tok_list: return None, tok - #print 'new_stat', string, set_vars, used_funcs, used_vars + #print 'new_stat', set_vars, used_funcs, used_vars if self.freshscope and not self.no_docstr and len(tok_list) == 1 \ and self.last_token[0] == tokenize.STRING: self.scope.add_docstr(self.last_token[1]) return None, tok else: - stmt = stmt_class(self.module, string, set_vars, used_funcs, + stmt = stmt_class(self.module, set_vars, used_funcs, used_vars, tok_list, first_pos, self.end_pos) self._check_user_stmt(stmt) @@ -667,7 +649,6 @@ class Parser(object): n, token_type, tok = self._parse_dot_name() if n: statement.set_vars.append(n) - statement.code += ',' + n.get_code() if statement: inputs.append(statement) first = False diff --git a/jedi/parsing_representation.py b/jedi/parsing_representation.py index 688def92..5ab61580 100644 --- a/jedi/parsing_representation.py +++ b/jedi/parsing_representation.py @@ -649,9 +649,6 @@ class Statement(Simple): stores pretty much all the Python code, except functions, classes, imports, and flow functions like if, for, etc. - :param code: The full code of a statement. This is import, if one wants \ - to execute the code at some level. - :param code: str :param set_vars: The variables which are defined by the statement. :param set_vars: str :param used_funcs: The functions which are used by the statement. @@ -663,14 +660,12 @@ class Statement(Simple): :param start_pos: Position (line, column) of the Statement. :type start_pos: tuple(int, int) """ - __slots__ = ('used_funcs', 'code', 'token_list', 'used_vars', + __slots__ = ('used_funcs', 'token_list', 'used_vars', 'set_vars', '_commands', '_assignment_details') - def __init__(self, module, code, set_vars, used_funcs, used_vars, + def __init__(self, module, set_vars, used_funcs, used_vars, token_list, start_pos, end_pos, parent=None): super(Statement, self).__init__(module, start_pos, end_pos) - # TODO remove code -> much cleaner - self.code = code self.used_funcs = used_funcs self.used_vars = used_vars self.token_list = token_list @@ -847,8 +842,8 @@ class Statement(Simple): if not token_list: return None, tok - statement = Statement(self._sub_module, "XXX" + self.code, [], [], [], - token_list, start_pos, end_pos) + statement = Statement(self._sub_module, [], [], [], + token_list, start_pos, end_pos) statement.parent = self.parent return statement, tok @@ -932,9 +927,9 @@ class Param(Statement): __slots__ = ('position_nr', 'is_generated', 'annotation_stmt', 'parent_function') - def __init__(self, module, code, set_vars, used_funcs, used_vars, + def __init__(self, module, set_vars, used_funcs, used_vars, token_list, start_pos, end_pos): - super(Param, self).__init__(module, code, set_vars, used_funcs, + super(Param, self).__init__(module, set_vars, used_funcs, used_vars, token_list, start_pos, end_pos) # this is defined by the parser later on, not at the initialization From 9690cf3eafb561a419855a61f61fbda255210824 Mon Sep 17 00:00:00 2001 From: David Halter Date: Thu, 21 Feb 2013 21:30:57 +0430 Subject: [PATCH 72/83] removed merged Statement.used_funcs and used_vars (made no sense) --- jedi/evaluate_representation.py | 4 ++-- jedi/parsing.py | 15 +++++---------- jedi/parsing_representation.py | 21 +++++++++------------ 3 files changed, 16 insertions(+), 24 deletions(-) diff --git a/jedi/evaluate_representation.py b/jedi/evaluate_representation.py index 2244d1f6..197d8cc5 100644 --- a/jedi/evaluate_representation.py +++ b/jedi/evaluate_representation.py @@ -508,7 +508,7 @@ class Execution(Executable): arr.values = values key_stmts = [] for key in keys: - stmt = pr.Statement(self._sub_module, [], [], [], [], + stmt = pr.Statement(self._sub_module, [], [], [], start_pos, None) stmt._commands = [key] key_stmts.append(stmt) @@ -624,7 +624,7 @@ class Execution(Executable): old = stmt # generate a statement if it's not already one. module = builtin.Builtin.scope - stmt = pr.Statement(module, [], [], [], [], (0, 0), None) + stmt = pr.Statement(module, [], [], [], (0, 0), None) stmt._commands = [old] # *args diff --git a/jedi/parsing.py b/jedi/parsing.py index 4349afa8..ef5f004b 100644 --- a/jedi/parsing.py +++ b/jedi/parsing.py @@ -312,7 +312,6 @@ class Parser(object): :rtype: (Statement, str) """ set_vars = [] - used_funcs = [] used_vars = [] level = 0 # The level of parentheses @@ -421,7 +420,7 @@ class Parser(object): i = 0 tok_list, toks = tok_list[:-i], tok_list[-i:-1] - st = pr.Statement(self.module, [], [], [], + st = pr.Statement(self.module, [], [], toks, first_pos, self.end_pos) tok = pr.ListComprehension(st, middle, in_clause, @@ -434,11 +433,7 @@ class Parser(object): tok_list.pop() if n: tok_list.append(n) - if tok == '(': - # it must be a function - used_funcs.append(n) - else: - used_vars.append(n) + used_vars.append(n) continue elif tok.endswith('=') and tok not in ['>=', '<=', '==', '!=']: # there has been an assignement -> change vars @@ -457,14 +452,14 @@ class Parser(object): if not tok_list: return None, tok - #print 'new_stat', set_vars, used_funcs, used_vars + #print 'new_stat', set_vars, used_vars if self.freshscope and not self.no_docstr and len(tok_list) == 1 \ and self.last_token[0] == tokenize.STRING: self.scope.add_docstr(self.last_token[1]) return None, tok else: - stmt = stmt_class(self.module, set_vars, used_funcs, - used_vars, tok_list, first_pos, self.end_pos) + stmt = stmt_class(self.module, set_vars, used_vars, tok_list, + first_pos, self.end_pos) self._check_user_stmt(stmt) diff --git a/jedi/parsing_representation.py b/jedi/parsing_representation.py index 5ab61580..4a84545f 100644 --- a/jedi/parsing_representation.py +++ b/jedi/parsing_representation.py @@ -651,8 +651,6 @@ class Statement(Simple): :param set_vars: The variables which are defined by the statement. :param set_vars: str - :param used_funcs: The functions which are used by the statement. - :param used_funcs: str :param used_vars: The variables which are used by the statement. :param used_vars: str :param token_list: Token list which is also peppered with Name. @@ -660,16 +658,15 @@ class Statement(Simple): :param start_pos: Position (line, column) of the Statement. :type start_pos: tuple(int, int) """ - __slots__ = ('used_funcs', 'token_list', 'used_vars', + __slots__ = ('token_list', 'used_vars', 'set_vars', '_commands', '_assignment_details') - def __init__(self, module, set_vars, used_funcs, used_vars, - token_list, start_pos, end_pos, parent=None): + def __init__(self, module, set_vars, used_vars, token_list, + start_pos, end_pos, parent=None): super(Statement, self).__init__(module, start_pos, end_pos) - self.used_funcs = used_funcs self.used_vars = used_vars self.token_list = token_list - for s in set_vars + used_funcs + used_vars: + for s in set_vars + used_vars: s.parent = self.use_as_parent self.set_vars = self._remove_executions_from_set_vars(set_vars) self.parent = parent @@ -842,7 +839,7 @@ class Statement(Simple): if not token_list: return None, tok - statement = Statement(self._sub_module, [], [], [], + statement = Statement(self._sub_module, [], [], token_list, start_pos, end_pos) statement.parent = self.parent return statement, tok @@ -927,10 +924,10 @@ class Param(Statement): __slots__ = ('position_nr', 'is_generated', 'annotation_stmt', 'parent_function') - def __init__(self, module, set_vars, used_funcs, used_vars, - token_list, start_pos, end_pos): - super(Param, self).__init__(module, set_vars, used_funcs, - used_vars, token_list, start_pos, end_pos) + def __init__(self, module, set_vars, used_vars, token_list, + start_pos, end_pos): + super(Param, self).__init__(module, set_vars, used_vars, token_list, + start_pos, end_pos) # this is defined by the parser later on, not at the initialization # it is the position in the call (first argument, second...) From 80423ddb57adb13bc72d8c3761f093686a2862ee Mon Sep 17 00:00:00 2001 From: David Halter Date: Thu, 21 Feb 2013 21:38:31 +0430 Subject: [PATCH 73/83] todo remove --- jedi/parsing_representation.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/jedi/parsing_representation.py b/jedi/parsing_representation.py index 4a84545f..200b60cf 100644 --- a/jedi/parsing_representation.py +++ b/jedi/parsing_representation.py @@ -14,9 +14,6 @@ One special thing: ``[1, 2+33]`` for example would be an Array with two ``Statement`` inside. This is the easiest way to write a parser. The same behaviour applies to ``Param``, which is being used in a function definition. - - -.. todo:: remove docstr params from Scope.__init__() """ import os @@ -112,8 +109,6 @@ class Scope(Simple, IsScope): :param start_pos: The position (line and column) of the scope. :type start_pos: tuple(int, int) - :param docstr: The docstring for the current Scope. - :type docstr: str """ def __init__(self, module, start_pos): super(Scope, self).__init__(module, start_pos) @@ -358,8 +353,6 @@ class Function(Scope): :type params: list :param start_pos: The start position (line, column) the Function. :type start_pos: tuple(int, int) - :param docstr: The docstring for the current Scope. - :type docstr: str """ def __init__(self, module, name, params, start_pos, annotation): super(Function, self).__init__(module, start_pos) From 923e59b9c2e9d3bcc774f852031fccb75f21d45a Mon Sep 17 00:00:00 2001 From: David Halter Date: Thu, 21 Feb 2013 21:45:39 +0430 Subject: [PATCH 74/83] fixed position problems (with tokenizer offsets) --- jedi/common.py | 10 ++++++++-- jedi/parsing_representation.py | 1 - 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/jedi/common.py b/jedi/common.py index cddac5d3..35a10f62 100644 --- a/jedi/common.py +++ b/jedi/common.py @@ -63,6 +63,7 @@ class NoErrorTokenizer(object): self.stop_on_scope = stop_on_scope self.first_scope = False self.closed = False + self.first = True def push_last_back(self): self.gen.push_back(self.current) @@ -107,8 +108,13 @@ class NoErrorTokenizer(object): elif c[1] != '@': self.first_scope = True - c[2] = self.offset[0] + c[2][0], self.offset[1] + c[2][1] - c[3] = self.offset[0] + c[3][0], self.offset[1] + c[3][1] + if self.first: + c[2] = self.offset[0] + c[2][0], self.offset[1] + c[2][1] + c[3] = self.offset[0] + c[3][0], self.offset[1] + c[3][1] + self.first = False + else: + c[2] = self.offset[0] + c[2][0], c[2][1] + c[3] = self.offset[0] + c[3][0], c[3][1] return c diff --git a/jedi/parsing_representation.py b/jedi/parsing_representation.py index 200b60cf..a886b319 100644 --- a/jedi/parsing_representation.py +++ b/jedi/parsing_representation.py @@ -90,7 +90,6 @@ class Simple(Base): def __repr__(self): code = self.get_code().replace('\n', ' ') - # TODO (this todo doesn't belong here) positions are not added right. return "<%s: %s@%s,%s>" % \ (type(self).__name__, code, self.start_pos[0], self.start_pos[1]) From f8e0c78f7741e3f5f2e6d804b7428103437b43e5 Mon Sep 17 00:00:00 2001 From: David Halter Date: Thu, 21 Feb 2013 22:04:35 +0430 Subject: [PATCH 75/83] Script.get_in_function_call -> Script.function_definition --- jedi/api.py | 18 +++++++++--- jedi/api_classes.py | 6 ++-- jedi/cache.py | 4 +-- jedi/settings.py | 16 +++++----- test/base.py | 4 +-- test/regression.py | 72 ++++++++++++++++++++++----------------------- 6 files changed, 66 insertions(+), 54 deletions(-) diff --git a/jedi/api.py b/jedi/api.py index 79b3c066..c3bda853 100644 --- a/jedi/api.py +++ b/jedi/api.py @@ -11,6 +11,7 @@ __all__ = ['Script', 'NotFoundError', 'set_debug_function', '_quick_complete'] import re import os +import warnings import parsing import parsing_representation as pr @@ -122,7 +123,7 @@ class Script(object): completions.append((c, s)) if not dot: # named params have no dots - call_def = self.get_in_function_call() + call_def = self.function_definition() if call_def: if not call_def.module.is_builtin(): for p in call_def.params: @@ -336,6 +337,15 @@ class Script(object): return sorted(set(names), key=lambda x: (x.module_path, x.start_pos)) def get_in_function_call(self): + """ + .. deprecated:: 0.5.0 + Use :attr:`.function_definition` instead. + .. todo:: Remove! + """ + warnings.warn("Use line instead.", DeprecationWarning) + return self.function_definition() + + def function_definition(self): """ Return the function object of the call you're currently in. @@ -391,7 +401,7 @@ class Script(object): debug.speed('func_call start') call = None - if settings.use_get_in_function_call_cache: + if settings.use_function_definition_cache: try: call, index = check_cache() except NotFoundError: @@ -405,9 +415,9 @@ class Script(object): return None debug.speed('func_call parsed') - with common.scale_speed_settings(settings.scale_get_in_function_call): + with common.scale_speed_settings(settings.scale_function_definition): _callable = lambda: evaluate.follow_call(call) - origins = cache.cache_get_in_function_call(_callable, user_stmt) + origins = cache.cache_function_definition(_callable, user_stmt) debug.speed('func_call followed') if len(origins) == 0: diff --git a/jedi/api_classes.py b/jedi/api_classes.py index 66d4db8c..35c3d76b 100644 --- a/jedi/api_classes.py +++ b/jedi/api_classes.py @@ -336,9 +336,11 @@ class RelatedName(BaseDefinition): class CallDef(object): - """ `CallDef` objects is the return value of `Script.get_in_function_call`. + """ + `CallDef` objects is the return value of `Script.function_definition`. It knows what functions you are currently in. e.g. `isinstance(` would - return the `isinstance` function. without `(` it would return nothing.""" + return the `isinstance` function. without `(` it would return nothing. + """ def __init__(self, executable, index, call): self.executable = executable self.index = index diff --git a/jedi/cache.py b/jedi/cache.py index f916beca..e60583f4 100644 --- a/jedi/cache.py +++ b/jedi/cache.py @@ -135,8 +135,8 @@ def time_cache(time_add_setting): return _temp -@time_cache("get_in_function_call_validity") -def cache_get_in_function_call(stmt): +@time_cache("function_definition_validity") +def cache_function_definition(stmt): module_path = stmt.get_parent_until().path return None if module_path is None else (module_path, stmt.start_pos) diff --git a/jedi/settings.py b/jedi/settings.py index 63c6413b..d8162170 100644 --- a/jedi/settings.py +++ b/jedi/settings.py @@ -34,7 +34,7 @@ Parser .. autodata:: fast_parser .. autodata:: fast_parser_always_reparse -.. autodata:: use_get_in_function_call_cache +.. autodata:: use_function_definition_cache Dynamic stuff @@ -66,14 +66,14 @@ definitely worse in some cases. But a completion should also be fast. .. autodata:: max_function_recursion_level .. autodata:: max_executions_without_builtins .. autodata:: max_executions -.. autodata:: scale_get_in_function_call +.. autodata:: scale_function_definition Caching ~~~~~~~ .. autodata:: star_import_cache_validity -.. autodata:: get_in_function_call_validity +.. autodata:: function_definition_validity Various @@ -156,9 +156,9 @@ This is just a debugging option. Always reparsing means that the fast parser is basically useless. So don't use it. """ -use_get_in_function_call_cache = True +use_function_definition_cache = True """ -Use the cache (full cache) to generate get_in_function_call's. This may fail +Use the cache (full cache) to generate function_definition's. This may fail with multiline docstrings (likely) and other complicated changes (unlikely). The goal is to move away from it by making the rest faster. """ @@ -225,9 +225,9 @@ max_executions = 250 A maximum amount of time, the completion may use. """ -scale_get_in_function_call = 0.1 +scale_function_definition = 0.1 """ -Because get_in_function_call is normally used on every single key hit, it has +Because function_definition is normally used on every single key hit, it has to be faster than a normal completion. This is the factor that is used to scale `max_executions` and `max_until_execution_unique`: """ @@ -253,7 +253,7 @@ might be slow, therefore we do a star import caching, that lasts a certain time span (in seconds). """ -get_in_function_call_validity = 3.0 +function_definition_validity = 3.0 """ Finding function calls might be slow (0.1-0.5s). This is not acceptible for normal writing. Therefore cache it for a short time. diff --git a/test/base.py b/test/base.py index 21ad78d7..fc78c437 100644 --- a/test/base.py +++ b/test/base.py @@ -65,9 +65,9 @@ class TestBase(unittest.TestCase): script = self.get_script(src, pos) return script.goto() - def get_in_function_call(self, src, pos=None): + def function_definition(self, src, pos=None): script = self.get_script(src, pos) - return script.get_in_function_call() + return script.function_definition() def print_summary(): print('\nSummary: (%s fails of %s tests) in %.3fs' % \ diff --git a/test/regression.py b/test/regression.py index c67c4e7a..97e9f37b 100755 --- a/test/regression.py +++ b/test/regression.py @@ -35,11 +35,11 @@ class TestRegression(TestBase): self.assertEqual(length, 1) def test_part_parser(self): - """ test the get_in_function_call speedups """ + """ test the function_definition speedups """ s = '\n' * 100 + 'abs(' pos = 101, 4 - self.get_in_function_call(s, pos) - assert self.get_in_function_call(s, pos) + self.function_definition(s, pos) + assert self.function_definition(s, pos) def test_get_definition_cursor(self): @@ -123,7 +123,7 @@ class TestRegression(TestBase): assert self.complete("from datetime import")[0].word == 'import' assert self.complete("from datetime import ") - def test_get_in_function_call(self): + def test_function_definition(self): def check(call_def, name, index): return call_def and call_def.call_name == name \ and call_def.index == index @@ -139,54 +139,54 @@ class TestRegression(TestBase): s7 = "str().upper().center(" s8 = "str(int[zip(" - assert check(self.get_in_function_call(s, (1, 4)), 'abs', 0) - assert check(self.get_in_function_call(s, (1, 6)), 'abs', 1) - assert check(self.get_in_function_call(s, (1, 7)), 'abs', 1) - assert check(self.get_in_function_call(s, (1, 8)), 'abs', 1) - assert check(self.get_in_function_call(s, (1, 11)), 'str', 0) + assert check(self.function_definition(s, (1, 4)), 'abs', 0) + assert check(self.function_definition(s, (1, 6)), 'abs', 1) + assert check(self.function_definition(s, (1, 7)), 'abs', 1) + assert check(self.function_definition(s, (1, 8)), 'abs', 1) + assert check(self.function_definition(s, (1, 11)), 'str', 0) - assert check(self.get_in_function_call(s2, (1, 4)), 'abs', 0) - assert self.get_in_function_call(s2, (1, 5)) is None - assert self.get_in_function_call(s2) is None + assert check(self.function_definition(s2, (1, 4)), 'abs', 0) + assert self.function_definition(s2, (1, 5)) is None + assert self.function_definition(s2) is None - assert self.get_in_function_call(s3, (1, 5)) is None - assert self.get_in_function_call(s3) is None + assert self.function_definition(s3, (1, 5)) is None + assert self.function_definition(s3) is None - assert self.get_in_function_call(s4, (1, 3)) is None - assert check(self.get_in_function_call(s4, (1, 4)), 'abs', 0) - assert check(self.get_in_function_call(s4, (1, 8)), 'zip', 0) - assert check(self.get_in_function_call(s4, (1, 9)), 'abs', 0) - #assert check(self.get_in_function_call(s4, (1, 10)), 'abs', 1) + assert self.function_definition(s4, (1, 3)) is None + assert check(self.function_definition(s4, (1, 4)), 'abs', 0) + assert check(self.function_definition(s4, (1, 8)), 'zip', 0) + assert check(self.function_definition(s4, (1, 9)), 'abs', 0) + #assert check(self.function_definition(s4, (1, 10)), 'abs', 1) - assert check(self.get_in_function_call(s5, (1, 4)), 'abs', 0) - assert check(self.get_in_function_call(s5, (1, 6)), 'abs', 1) + assert check(self.function_definition(s5, (1, 4)), 'abs', 0) + assert check(self.function_definition(s5, (1, 6)), 'abs', 1) - assert check(self.get_in_function_call(s6), 'center', 0) - assert check(self.get_in_function_call(s6, (1, 4)), 'str', 0) + assert check(self.function_definition(s6), 'center', 0) + assert check(self.function_definition(s6, (1, 4)), 'str', 0) - assert check(self.get_in_function_call(s7), 'center', 0) - assert check(self.get_in_function_call(s8), 'zip', 0) - assert check(self.get_in_function_call(s8, (1, 8)), 'str', 0) + assert check(self.function_definition(s7), 'center', 0) + assert check(self.function_definition(s8), 'zip', 0) + assert check(self.function_definition(s8, (1, 8)), 'str', 0) s = "import time; abc = time; abc.sleep(" - assert check(self.get_in_function_call(s), 'sleep', 0) + assert check(self.function_definition(s), 'sleep', 0) # jedi-vim #9 s = "with open(" - assert check(self.get_in_function_call(s), 'open', 0) + assert check(self.function_definition(s), 'open', 0) # jedi-vim #11 s1 = "for sorted(" - assert check(self.get_in_function_call(s1), 'sorted', 0) + assert check(self.function_definition(s1), 'sorted', 0) s2 = "for s in sorted(" - assert check(self.get_in_function_call(s2), 'sorted', 0) + assert check(self.function_definition(s2), 'sorted', 0) # jedi #57 s = "def func(alpha, beta): pass\n" \ "func(alpha='101'," - assert check(self.get_in_function_call(s, (2, 13)), 'func', 0) + assert check(self.function_definition(s, (2, 13)), 'func', 0) - def test_get_in_function_call_complex(self): + def test_function_definition_complex(self): def check(call_def, name, index): return call_def and call_def.call_name == name \ and call_def.index == index @@ -201,17 +201,17 @@ class TestRegression(TestBase): if 1: pass """ - assert check(self.get_in_function_call(s, (6, 24)), 'abc', 0) + assert check(self.function_definition(s, (6, 24)), 'abc', 0) s = """ import re def huhu(it): re.compile( return it * 2 """ - assert check(self.get_in_function_call(s, (4, 31)), 'compile', 0) + assert check(self.function_definition(s, (4, 31)), 'compile', 0) # jedi-vim #70 s = """def foo(""" - assert self.get_in_function_call(s) is None + assert self.function_definition(s) is None def test_add_dynamic_mods(self): api.settings.additional_dynamic_modules = ['dynamic.py'] @@ -389,7 +389,7 @@ class TestSpeed(TestBase): def test_scipy_speed(self): s = 'import scipy.weave; scipy.weave.inline(' script = jedi.Script(s, 1, len(s), '') - script.get_in_function_call() + script.function_definition() #print(jedi.imports.imports_processed) if __name__ == '__main__': From b0c3fd54394d569d280c9143dc81368eff0519a8 Mon Sep 17 00:00:00 2001 From: David Halter Date: Thu, 21 Feb 2013 22:12:51 +0430 Subject: [PATCH 76/83] api.Script.get_definition -> definition, to be consistent in the api naming. deprecated api.Script.get_definition --- jedi/api.py | 13 +++++++++++-- jedi/api_classes.py | 2 +- test/base.py | 8 +++++--- test/regression.py | 39 +++++++++++++++++++-------------------- test/run.py | 10 +++++----- 5 files changed, 41 insertions(+), 31 deletions(-) diff --git a/jedi/api.py b/jedi/api.py index c3bda853..394df66b 100644 --- a/jedi/api.py +++ b/jedi/api.py @@ -170,7 +170,7 @@ class Script(object): x.word.lower())) def _prepare_goto(self, goto_path, is_like_search=False): - """ Base for complete, goto and get_definition. Basically it returns + """ Base for complete, goto and definition. Basically it returns the resolved scopes under cursor. """ debug.dbg('start: %s in %s' % (goto_path, self._parser.user_scope)) @@ -201,11 +201,20 @@ class Script(object): return stmt def get_definition(self): + """ + .. deprecated:: 0.5.0 + Use :attr:`.function_definition` instead. + .. todo:: Remove! + """ + warnings.warn("Use line instead.", DeprecationWarning) + return self.definition() + + def definition(self): """ Return the definitions of a the path under the cursor. This is not a goto function! This follows complicated paths and returns the end, not the first definition. The big difference between :meth:`goto` and - :meth:`get_definition` is that :meth:`goto` doesn't follow imports and + :meth:`definition` is that :meth:`goto` doesn't follow imports and statements. Multiple objects may be returned, because Python itself is a dynamic language, which means depending on an option you can have two different versions of a function. diff --git a/jedi/api_classes.py b/jedi/api_classes.py index 35c3d76b..a30ea147 100644 --- a/jedi/api_classes.py +++ b/jedi/api_classes.py @@ -263,7 +263,7 @@ class Completion(BaseDefinition): class Definition(BaseDefinition): """ *Definition* objects are returned from :meth:`api.Script.goto` or - :meth:`api.Script.get_definition`. + :meth:`api.Script.definition`. """ def __init__(self, definition): super(Definition, self).__init__(definition, definition.start_pos) diff --git a/test/base.py b/test/base.py index fc78c437..f78e939b 100644 --- a/test/base.py +++ b/test/base.py @@ -32,6 +32,7 @@ sys.argv = sys.argv[:1] + args summary = [] tests_fail = 0 + def get_test_list(): # get test list, that should be executed test_files = {} @@ -46,6 +47,7 @@ def get_test_list(): last = arg return test_files + class TestBase(unittest.TestCase): def get_script(self, src, pos, path=None): if pos is None: @@ -53,9 +55,9 @@ class TestBase(unittest.TestCase): pos = len(lines), len(lines[-1]) return jedi.Script(src, pos[0], pos[1], path) - def get_def(self, src, pos=None): + def definition(self, src, pos=None): script = self.get_script(src, pos) - return script.get_definition() + return script.definition() def complete(self, src, pos=None, path=None): script = self.get_script(src, pos, path) @@ -69,9 +71,9 @@ class TestBase(unittest.TestCase): script = self.get_script(src, pos) return script.function_definition() + def print_summary(): print('\nSummary: (%s fails of %s tests) in %.3fs' % \ (tests_fail, test_sum, time.time() - t_start)) for s in summary: print(s) - diff --git a/test/regression.py b/test/regression.py index 97e9f37b..fdf431f9 100755 --- a/test/regression.py +++ b/test/regression.py @@ -41,7 +41,7 @@ class TestRegression(TestBase): self.function_definition(s, pos) assert self.function_definition(s, pos) - def test_get_definition_cursor(self): + def test_definition_cursor(self): s = ("class A():\n" " def _something(self):\n" @@ -60,7 +60,7 @@ class TestRegression(TestBase): diff_line = 4, 10 should2 = 8, 10 - get_def = lambda pos: [d.description for d in self.get_def(s, pos)] + get_def = lambda pos: [d.description for d in self.definition(s, pos)] in_name = get_def(in_name) under_score = get_def(under_score) should1 = get_def(should1) @@ -71,32 +71,31 @@ class TestRegression(TestBase): assert should1 == in_name assert should1 == under_score - #print should2, diff_line assert should2 == diff_line self.assertRaises(jedi.NotFoundError, get_def, cls) def test_keyword_doc(self): - r = list(self.get_def("or", (1, 1))) + r = list(self.definition("or", (1, 1))) assert len(r) == 1 if not is_py25: assert len(r[0].doc) > 100 - r = list(self.get_def("asfdasfd", (1, 1))) + r = list(self.definition("asfdasfd", (1, 1))) assert len(r) == 0 def test_operator_doc(self): - r = list(self.get_def("a == b", (1, 3))) + r = list(self.definition("a == b", (1, 3))) assert len(r) == 1 if not is_py25: assert len(r[0].doc) > 100 - def test_get_definition_at_zero(self): - assert self.get_def("a", (1, 1)) == [] - s = self.get_def("str", (1, 1)) + def test_definition_at_zero(self): + assert self.definition("a", (1, 1)) == [] + s = self.definition("str", (1, 1)) assert len(s) == 1 assert list(s)[0].description == 'class str' - assert self.get_def("", (1, 0)) == [] + assert self.definition("", (1, 0)) == [] def test_complete_at_zero(self): s = self.complete("str", (1, 3)) @@ -106,9 +105,9 @@ class TestRegression(TestBase): s = self.complete("", (1, 0)) assert len(s) > 0 - def test_get_definition_on_import(self): - assert self.get_def("import sys_blabla", (1, 8)) == [] - assert len(self.get_def("import sys", (1, 8))) == 1 + def test_definition_on_import(self): + assert self.definition("import sys_blabla", (1, 8)) == [] + assert len(self.definition("import sys", (1, 8))) == 1 def test_complete_on_empty_import(self): # should just list the files in the directory @@ -222,15 +221,15 @@ class TestRegression(TestBase): # .parser to load the module api.modules.Module(os.path.abspath('dynamic.py'), src2).parser script = jedi.Script(src1, 1, len(src1), '../setup.py') - result = script.get_definition() + result = script.definition() assert len(result) == 1 assert result[0].description == 'class int' def test_named_import(self): """ named import - jedi-vim issue #8 """ s = "import time as dt" - assert len(jedi.Script(s, 1, 15, '/').get_definition()) == 1 - assert len(jedi.Script(s, 1, 10, '/').get_definition()) == 1 + assert len(jedi.Script(s, 1, 15, '/').definition()) == 1 + assert len(jedi.Script(s, 1, 10, '/').definition()) == 1 def test_unicode_script(self): """ normally no unicode objects are being used. (<=2.7) """ @@ -275,10 +274,10 @@ class TestRegression(TestBase): def test_keyword_definition_doc(self): """ github jedi-vim issue #44 """ - defs = self.get_def("print") + defs = self.definition("print") assert [d.doc for d in defs] - defs = self.get_def("import") + defs = self.definition("import") assert len(defs) == 1 assert [d.doc for d in defs] @@ -325,7 +324,7 @@ class TestFeature(TestBase): assert self.complete('import os; os.path.join')[0].full_name \ == 'os.path.join' # issue #94 - defs = self.get_def("""import os; os.path.join(""") + defs = self.definition("""import os; os.path.join(""") assert defs[0].full_name is None def test_full_name_builtin(self): @@ -336,7 +335,7 @@ class TestFeature(TestBase): import re any_re = re.compile('.*') any_re""" - self.assertEqual(self.get_def(s)[0].full_name, 're.RegexObject') + self.assertEqual(self.definition(s)[0].full_name, 're.RegexObject') def test_quick_completion(self): sources = [ diff --git a/test/run.py b/test/run.py index 67c527f8..b9571a80 100755 --- a/test/run.py +++ b/test/run.py @@ -36,7 +36,7 @@ def run_definition_test(script, should_str, line_nr): Runs tests for definitions. Return if the test was a fail or not, with 1 for fail and 0 for success. """ - result = script.get_definition() + result = script.definition() is_str = set(r.desc_with_module for r in result) if is_str != should_str: print('Solution @%s not right, received %s, wanted %s' \ @@ -120,10 +120,10 @@ def run_test(source, f_name, lines_to_execute): >>> #? int() >>> ab = 3; ab """ - def get_defs(correct, correct_start, path): + def definition(correct, correct_start, path): def defs(line_nr, indent): script = jedi.Script(source, line_nr, indent, path) - return set(script.get_definition()) + return set(script.definition()) should_be = set() number = 0 @@ -166,7 +166,7 @@ def run_test(source, f_name, lines_to_execute): else: index = len(line) - 1 # -1 for the \n # if a list is wanted, use the completion test, otherwise the - # get_definition test + # definition test path = completion_test_dir + os.path.sep + f_name try: script = jedi.Script(source, line_nr, index, path) @@ -177,7 +177,7 @@ def run_test(source, f_name, lines_to_execute): elif correct.startswith('['): fails += run_completion_test(script, correct, line_nr) else: - should_str = get_defs(correct, start, path) + should_str = definition(correct, start, path) fails += run_definition_test(script, should_str, line_nr) except Exception: print(traceback.format_exc()) From 2fda713118729ece0bd62e0ec0646f67577cd22c Mon Sep 17 00:00:00 2001 From: David Halter Date: Fri, 22 Feb 2013 23:02:44 +0430 Subject: [PATCH 77/83] basic listcomprehension/lambda move --- jedi/parsing.py | 90 +++------------------ jedi/parsing_representation.py | 143 +++++++++++++++++++++++++++++---- jedi/recursion.py | 2 +- 3 files changed, 138 insertions(+), 97 deletions(-) diff --git a/jedi/parsing.py b/jedi/parsing.py index ef5f004b..7b099a51 100644 --- a/jedi/parsing.py +++ b/jedi/parsing.py @@ -297,7 +297,7 @@ class Parser(object): return scope def _parse_statement(self, pre_used_token=None, added_breaks=None, - stmt_class=pr.Statement, list_comp=False): + stmt_class=pr.Statement): """ Parses statements like: @@ -355,86 +355,16 @@ class Parser(object): set_vars.append(n) tok_list.append(n) continue - elif tok == 'lambda': - params = [] - start_pos = self.start_pos - while tok != ':': - param, tok = self._parse_statement( - added_breaks=[':', ','], stmt_class=pr.Param) - if param is None: - break - params.append(param) - if tok != ':': - continue - - lambd = pr.Lambda(self.module, params, start_pos) - ret, tok = self._parse_statement(added_breaks=[',']) - if ret is not None: - ret.parent = lambd - lambd.returns.append(ret) - lambd.parent = self.scope - lambd.end_pos = self.end_pos - tok_list[-1] = lambd - continue + elif tok in ['lambda', 'for', 'in']: + pass # don't parse these keywords, parse later in stmt. elif token_type == tokenize.NAME: - if tok == 'for': - # list comprehensions! - middle, tok = self._parse_statement( - added_breaks=['in']) - if tok != 'in' or middle is None: - if middle is None: - level -= 1 - else: - middle.parent = self.scope - debug.warning('list comprehension formatting @%s' % - self.start_pos[0]) - continue - - b = [')', ']'] - in_clause, tok = self._parse_statement(added_breaks=b, - list_comp=True) - if tok not in b or in_clause is None: - middle.parent = self.scope - if in_clause is None: - self._gen.push_last_back() - else: - in_clause.parent = self.scope - in_clause.parent = self.scope - debug.warning('list comprehension in_clause %s@%s' - % (repr(tok), self.start_pos[0])) - continue - other_level = 0 - - for i, tok in enumerate(reversed(tok_list)): - if not isinstance(tok, (pr.Name, - pr.ListComprehension)): - tok = tok[1] - if tok in closing_brackets: - other_level -= 1 - elif tok in opening_brackets: - other_level += 1 - if other_level > 0: - break - else: - # could not detect brackets -> nested list comp - i = 0 - - tok_list, toks = tok_list[:-i], tok_list[-i:-1] - st = pr.Statement(self.module, [], [], - toks, first_pos, self.end_pos) - - tok = pr.ListComprehension(st, middle, in_clause, - self.scope) - tok_list.append(tok) - continue - else: - n, token_type, tok = self._parse_dot_name(self.current) - # removed last entry, because we add Name - tok_list.pop() - if n: - tok_list.append(n) - used_vars.append(n) - continue + n, token_type, tok = self._parse_dot_name(self.current) + # removed last entry, because we add Name + tok_list.pop() + if n: + tok_list.append(n) + used_vars.append(n) + continue elif tok.endswith('=') and tok not in ['>=', '<=', '==', '!=']: # there has been an assignement -> change vars if level == 0: diff --git a/jedi/parsing_representation.py b/jedi/parsing_representation.py index a886b319..cd3e6799 100644 --- a/jedi/parsing_representation.py +++ b/jedi/parsing_representation.py @@ -499,9 +499,10 @@ class Flow(Scope): """ Get the names for the flow. This includes also a call to the super class. - :param is_internal_call: defines an option for internal files to crawl\ - through this class. Normally it will just call its superiors, to\ - generate the output. + + :param is_internal_call: defines an option for internal files to crawl + through this class. Normally it will just call its superiors, to + generate the output. """ if is_internal_call: n = list(self.set_vars) @@ -521,7 +522,7 @@ class Flow(Scope): return i def set_next(self, next): - """ Set the next element in the flow, those are else, except, etc. """ + """Set the next element in the flow, those are else, except, etc.""" if self.next: return self.next.set_next(next) else: @@ -756,7 +757,8 @@ class Statement(Simple): return isinstance(tok, (str, unicode)) and tok.endswith('=') \ and not tok in ['>=', '<=', '==', '!='] - def parse_array(token_iterator, array_type, start_pos, add_el=None): + def parse_array(token_iterator, array_type, start_pos, add_el=None, + added_breaks=()): arr = Array(self._sub_module, start_pos, array_type, self) if add_el is not None: arr.add_statement(add_el) @@ -765,8 +767,9 @@ class Statement(Simple): break_tok = None is_array = None while True: - stmt, break_tok = parse_array_el(token_iterator, maybe_dict, - break_on_assignment=bool(add_el)) + stmt, break_tok = parse_stmt(token_iterator, maybe_dict, + break_on_assignment=bool(add_el), + added_breaks=added_breaks) if stmt is None: break else: @@ -774,7 +777,9 @@ class Statement(Simple): is_array = True is_key = maybe_dict and break_tok == ':' arr.add_statement(stmt, is_key) - if break_tok in closing_brackets or is_assignment(break_tok): + if break_tok in closing_brackets \ + or break_tok in added_breaks \ + or is_assignment(break_tok): break if arr.type == Array.TUPLE and len(arr) == 1 and not is_array: arr.type = Array.NOARRAY @@ -791,12 +796,14 @@ class Statement(Simple): else 0) return arr, break_tok - def parse_array_el(token_iterator, maybe_dict=False, - break_on_assignment=False): + def parse_stmt(token_iterator, maybe_dict=False, added_breaks=(), + break_on_assignment=False, stmt_class=Statement): token_list = [] + used_vars = [] level = 1 tok = None first = True + end_pos = None for i, tok_temp in token_iterator: if isinstance(tok_temp, Base): # the token is a Name, which has already been parsed @@ -808,20 +815,34 @@ class Statement(Simple): if isinstance(tok, ListComprehension): # it's not possible to set it earlier tok.parent = self + if isinstance(tok, Name): + used_vars.append(tok) else: token_type, tok, start_tok_pos = tok_temp + last_end_pos = end_pos end_pos = start_tok_pos[0], start_tok_pos[1] + len(tok) if first: first = False start_pos = start_tok_pos + if tok == 'lambda': + lambd = parse_lambda(token_iterator) + if lambd is not None: + token_list.append(lambd) + elif tok == 'for': + list_comp, tok = parse_list_comp(token_iterator, + token_list, start_pos, last_end_pos) + if list_comp is not None: + token_list = [list_comp] + if tok in closing_brackets: level -= 1 elif tok in brackets.keys(): level += 1 if level == 0 and tok in closing_brackets \ - or level == 1 and (tok == ',' + or tok in added_breaks \ + or level == 1 and (tok == ',' or maybe_dict and tok == ':' or is_assignment(tok) and break_on_assignment): end_pos = end_pos[0], end_pos[1] - 1 @@ -831,11 +852,101 @@ class Statement(Simple): if not token_list: return None, tok - statement = Statement(self._sub_module, [], [], - token_list, start_pos, end_pos) - statement.parent = self.parent + statement = Statement(self._sub_module, [], [], token_list, + start_pos, end_pos, self.parent) + statement.used_vars = used_vars return statement, tok + def parse_lambda(token_iterator): + params = [] + start_pos = self.start_pos + tok = next(token_iterator) + while tok != ':': + param, tok = parse_stmt(token_iterator, + added_breaks=[':', ','], stmt_class=Param) + if param is None: + break + params.append(param) + if tok != ':': + return None, tok + + lambd = Lambda(self.module, params, start_pos) + ret, tok = self._parse_statement(added_breaks=[',']) + if ret is not None: + ret.parent = lambd + lambd.returns.append(ret) + lambd.parent = self.scope + lambd.end_pos = self.end_pos + return lambd + + + def parse_list_comp(token_iterator, token_list, start_pos, end_pos): + def parse_stmt_or_arr(token_iterator, added_breaks=()): + stmt, tok = parse_stmt(token_iterator, added_breaks=added_breaks) + if tok == ',': + arr, tok = parse_array(token_iterator, Array.TUPLE, + stmt.start_pos, stmt, + added_breaks=added_breaks) + used_vars = [] + for stmt in arr: + used_vars += stmt.used_vars + start_pos = arr.start_pos[0], arr.start_pos[1] - 1 + stmt = Statement(self._sub_module, [], used_vars, [], + start_pos, arr.end_pos) + arr.parent = stmt + stmt.token_list = stmt._commands = [arr] + else: + for v in stmt.used_vars: + v.parent = stmt + return stmt, tok + + st = Statement(self._sub_module, [], [], token_list, start_pos, + end_pos) + + middle, tok = parse_stmt_or_arr(token_iterator, added_breaks=['in']) + if tok != 'in' or middle is None: + #if middle is None: + # level -= 1 + #else: + #middle.parent = self.scope + debug.warning('list comprehension formatting @%s' % + start_pos[0]) + return None, tok + + in_clause, tok = parse_stmt_or_arr(token_iterator) + """ + if tok not in b or in_clause is None: + #middle.parent = self.scope + if in_clause is None: + self._gen.push_last_back() + #else: + # in_clause.parent = self.scope + # in_clause.parent = self.scope + debug.warning('list comprehension in_clause %s@%s' + % (repr(tok), start_pos[0])) + return None, tok + """ + """ + other_level = 0 + + for i, tok in enumerate(reversed(token_list)): + if not isinstance(tok, (Name, ListComprehension)): + tok = tok[1] + if tok in closing_brackets: + other_level -= 1 + elif tok in brackets.keys(): + other_level += 1 + if other_level > 0: + break + else: + # could not detect brackets -> nested list comp + i = 0 +""" + #token_list, toks = token_list[:-i], token_list[-i:-1] + + + return ListComprehension(st, middle, in_clause, self), tok + # initializations result = [] is_chain = False @@ -860,7 +971,7 @@ class Statement(Simple): is_chain = False start = i + 1 continue - elif tok == 'as': # just ignore as + elif tok == 'as': # just ignore as, because it sets values next(token_iterator, None) continue @@ -893,7 +1004,7 @@ class Statement(Simple): is_chain = True elif tok == ',': # implies a tuple # rewrite `result`, because now the whole thing is a tuple - add_el, t = parse_array_el(enumerate(self.token_list[start:i])) + add_el, t = parse_stmt(enumerate(self.token_list[start:i])) arr, break_tok = parse_array(token_iterator, Array.TUPLE, add_el.start_pos, add_el) result = [arr] diff --git a/jedi/recursion.py b/jedi/recursion.py index a3a41006..e12a7ce4 100644 --- a/jedi/recursion.py +++ b/jedi/recursion.py @@ -35,7 +35,7 @@ class RecursionDecorator(object): def push_stmt(self, stmt): self.current = RecursionNode(stmt, self.current) check = self._check_recursion() - if check: + if check:# TODO remove False!!!! debug.warning('catched stmt recursion: %s against %s @%s' % (stmt, check.stmt, stmt.start_pos)) self.pop_stmt() From c1e805d7b02d415e023f31c9d233ec7f064c243c Mon Sep 17 00:00:00 2001 From: David Halter Date: Fri, 22 Feb 2013 23:06:59 +0430 Subject: [PATCH 78/83] fixed invalid list comprehension tests --- jedi/parsing_representation.py | 4 ++++ test/completion/invalid.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/jedi/parsing_representation.py b/jedi/parsing_representation.py index cd3e6799..5d7b1004 100644 --- a/jedi/parsing_representation.py +++ b/jedi/parsing_representation.py @@ -883,6 +883,8 @@ class Statement(Simple): def parse_list_comp(token_iterator, token_list, start_pos, end_pos): def parse_stmt_or_arr(token_iterator, added_breaks=()): stmt, tok = parse_stmt(token_iterator, added_breaks=added_breaks) + if not stmt: + return None, tok if tok == ',': arr, tok = parse_array(token_iterator, Array.TUPLE, stmt.start_pos, stmt, @@ -945,6 +947,8 @@ class Statement(Simple): #token_list, toks = token_list[:-i], token_list[-i:-1] + if middle is None or in_clause is None: + return None, tok return ListComprehension(st, middle, in_clause, self), tok # initializations diff --git a/test/completion/invalid.py b/test/completion/invalid.py index 14b00413..e755041e 100644 --- a/test/completion/invalid.py +++ b/test/completion/invalid.py @@ -101,7 +101,7 @@ a[0] a = [a for a in [1,2] def break(): pass -#? list() +#? int() a[0] #? [] From 7f051087e581cbc210663ce492ebc4956c27bf62 Mon Sep 17 00:00:00 2001 From: David Halter Date: Fri, 22 Feb 2013 23:37:59 +0430 Subject: [PATCH 79/83] basic lambda implementation --- jedi/parsing.py | 8 +++++--- jedi/parsing_representation.py | 35 ++++++++++++++++++++-------------- 2 files changed, 26 insertions(+), 17 deletions(-) diff --git a/jedi/parsing.py b/jedi/parsing.py index 7b099a51..c333ce85 100644 --- a/jedi/parsing.py +++ b/jedi/parsing.py @@ -333,12 +333,12 @@ class Parser(object): # will even break in parentheses. This is true for typical flow # commands like def and class and the imports, which will never be used # in a statement. - breaks = ['\n', ':', ')'] + breaks = set(['\n', ':', ')']) always_break = [';', 'import', 'from', 'class', 'def', 'try', 'except', 'finally', 'while', 'return', 'yield'] not_first_break = ['del', 'raise'] if added_breaks: - breaks += added_breaks + breaks |= set(added_breaks) tok_list = [] while not (tok in always_break @@ -356,7 +356,9 @@ class Parser(object): tok_list.append(n) continue elif tok in ['lambda', 'for', 'in']: - pass # don't parse these keywords, parse later in stmt. + # don't parse these keywords, parse later in stmt. + if tok == 'lambda': + breaks.remove(':') elif token_type == tokenize.NAME: n, token_type, tok = self._parse_dot_name(self.current) # removed last entry, because we add Name diff --git a/jedi/parsing_representation.py b/jedi/parsing_representation.py index 5d7b1004..a5b3278e 100644 --- a/jedi/parsing_representation.py +++ b/jedi/parsing_representation.py @@ -421,8 +421,9 @@ class Function(Scope): class Lambda(Function): - def __init__(self, module, params, start_pos): + def __init__(self, module, params, start_pos, parent): super(Lambda, self).__init__(module, None, params, start_pos, None) + self.parent = parent def get_code(self, first_indent=False, indention=' '): params = ','.join([stmt.get_code() for stmt in self.params]) @@ -852,7 +853,7 @@ class Statement(Simple): if not token_list: return None, tok - statement = Statement(self._sub_module, [], [], token_list, + statement = stmt_class(self._sub_module, [], [], token_list, start_pos, end_pos, self.parent) statement.used_vars = used_vars return statement, tok @@ -860,26 +861,28 @@ class Statement(Simple): def parse_lambda(token_iterator): params = [] start_pos = self.start_pos - tok = next(token_iterator) - while tok != ':': - param, tok = parse_stmt(token_iterator, - added_breaks=[':', ','], stmt_class=Param) + while True: + param, tok = parse_stmt(token_iterator, added_breaks=[':'], + stmt_class=Param) if param is None: break params.append(param) + if tok == ':': + break if tok != ':': - return None, tok + return None - lambd = Lambda(self.module, params, start_pos) - ret, tok = self._parse_statement(added_breaks=[',']) + # since lambda is a Function scope, it needs Scope parents + parent = self.get_parent_until(IsScope) + lambd = Lambda(self._sub_module, params, start_pos, parent) + + ret, tok = parse_stmt(token_iterator) if ret is not None: ret.parent = lambd lambd.returns.append(ret) - lambd.parent = self.scope lambd.end_pos = self.end_pos return lambd - def parse_list_comp(token_iterator, token_list, start_pos, end_pos): def parse_stmt_or_arr(token_iterator, added_breaks=()): stmt, tok = parse_stmt(token_iterator, added_breaks=added_breaks) @@ -980,7 +983,11 @@ class Statement(Simple): continue is_literal = token_type in [tokenize.STRING, tokenize.NUMBER] - if isinstance(tok, Name) or is_literal: + if tok == 'lambda': + lambd = parse_lambda(token_iterator) + if lambd is not None: + result.append(lambd) + elif isinstance(tok, Name) or is_literal: c_type = Call.NAME if is_literal: tok = literal_eval(tok) @@ -1032,9 +1039,9 @@ class Param(Statement): 'parent_function') def __init__(self, module, set_vars, used_vars, token_list, - start_pos, end_pos): + start_pos, end_pos, parent=None): super(Param, self).__init__(module, set_vars, used_vars, token_list, - start_pos, end_pos) + start_pos, end_pos, parent) # this is defined by the parser later on, not at the initialization # it is the position in the call (first argument, second...) From 6912764923ee65ce05fc18a50bd70083b1262c35 Mon Sep 17 00:00:00 2001 From: David Halter Date: Sat, 23 Feb 2013 00:40:34 +0430 Subject: [PATCH 80/83] fix position problems which came with lambdas --- jedi/parsing.py | 2 +- jedi/parsing_representation.py | 43 +++++++++++++++++++++++----------- 2 files changed, 30 insertions(+), 15 deletions(-) diff --git a/jedi/parsing.py b/jedi/parsing.py index c333ce85..95992b54 100644 --- a/jedi/parsing.py +++ b/jedi/parsing.py @@ -358,7 +358,7 @@ class Parser(object): elif tok in ['lambda', 'for', 'in']: # don't parse these keywords, parse later in stmt. if tok == 'lambda': - breaks.remove(':') + breaks.discard(':') elif token_type == tokenize.NAME: n, token_type, tok = self._parse_dot_name(self.current) # removed last entry, because we add Name diff --git a/jedi/parsing_representation.py b/jedi/parsing_representation.py index a5b3278e..c51e6e86 100644 --- a/jedi/parsing_representation.py +++ b/jedi/parsing_representation.py @@ -427,7 +427,7 @@ class Lambda(Function): def get_code(self, first_indent=False, indention=' '): params = ','.join([stmt.get_code() for stmt in self.params]) - string = "lambda %s:" % params + string = "lambda %s: " % params return string + super(Function, self).get_code(indention=indention) def __repr__(self): @@ -827,7 +827,7 @@ class Statement(Simple): start_pos = start_tok_pos if tok == 'lambda': - lambd = parse_lambda(token_iterator) + lambd, tok = parse_lambda(token_iterator) if lambd is not None: token_list.append(lambd) elif tok == 'for': @@ -870,7 +870,7 @@ class Statement(Simple): if tok == ':': break if tok != ':': - return None + return None, tok # since lambda is a Function scope, it needs Scope parents parent = self.get_parent_until(IsScope) @@ -881,7 +881,7 @@ class Statement(Simple): ret.parent = lambd lambd.returns.append(ret) lambd.end_pos = self.end_pos - return lambd + return lambd, tok def parse_list_comp(token_iterator, token_list, start_pos, end_pos): def parse_stmt_or_arr(token_iterator, added_breaks=()): @@ -959,9 +959,8 @@ class Statement(Simple): is_chain = False brackets = {'(': Array.TUPLE, '[': Array.LIST, '{': Array.SET} closing_brackets = ')', '}', ']' - start = 0 - token_iterator = enumerate(self.token_list) + token_iterator = common.PushBackIterator(enumerate(self.token_list)) for i, tok_temp in token_iterator: if isinstance(tok_temp, Base): # the token is a Name, which has already been parsed @@ -976,18 +975,18 @@ class Statement(Simple): self._assignment_details.append((result, tok)) result = [] is_chain = False - start = i + 1 continue elif tok == 'as': # just ignore as, because it sets values next(token_iterator, None) continue - is_literal = token_type in [tokenize.STRING, tokenize.NUMBER] if tok == 'lambda': - lambd = parse_lambda(token_iterator) + lambd, tok = parse_lambda(token_iterator) if lambd is not None: result.append(lambd) - elif isinstance(tok, Name) or is_literal: + + is_literal = token_type in [tokenize.STRING, tokenize.NUMBER] + if isinstance(tok, Name) or is_literal: c_type = Call.NAME if is_literal: tok = literal_eval(tok) @@ -1014,16 +1013,32 @@ class Statement(Simple): if result and isinstance(result[-1], Call): is_chain = True elif tok == ',': # implies a tuple - # rewrite `result`, because now the whole thing is a tuple - add_el, t = parse_stmt(enumerate(self.token_list[start:i])) + # commands is now an array not a statement anymore + t = result[0] + start_pos = t[2] if isinstance(t, tuple) else t.start_pos + + # get the correct index + i, tok = next(token_iterator, (len(self.token_list), None)) + if tok is not None: + token_iterator.push_back((i, tok)) + t = self.token_list[i - 1] + try: + end_pos = t.end_pos + except AttributeError: + end_pos = (t[2][0], t[2][1] + len(t[1])) \ + if isinstance(t, tuple) else t.start_pos + + stmt = Statement(self._sub_module, [], [], result, + start_pos, end_pos, self.parent) + stmt._commands = result + #add_el, t = parse_stmt(enumerate(self.token_list[start:i - 1])) arr, break_tok = parse_array(token_iterator, Array.TUPLE, - add_el.start_pos, add_el) + stmt.start_pos, stmt) result = [arr] if is_assignment(break_tok): self._assignment_details.append((result, break_tok)) result = [] is_chain = False - start = i + 1 else: if tok != '\n': result.append(tok) From 4d7aad4ce8444b94dc42d6cc59963a3b05d89a69 Mon Sep 17 00:00:00 2001 From: David Halter Date: Sat, 23 Feb 2013 01:11:29 +0430 Subject: [PATCH 81/83] deleted old crap code --- jedi/parsing_representation.py | 49 +++++++--------------------------- test/regression.py | 3 ++- 2 files changed, 12 insertions(+), 40 deletions(-) diff --git a/jedi/parsing_representation.py b/jedi/parsing_representation.py index c51e6e86..2237e680 100644 --- a/jedi/parsing_representation.py +++ b/jedi/parsing_representation.py @@ -21,7 +21,7 @@ import re import tokenize from _compatibility import next, literal_eval, cleandoc, Python3Method, \ - property, unicode + property, unicode, is_py3k import common import debug @@ -910,48 +910,14 @@ class Statement(Simple): middle, tok = parse_stmt_or_arr(token_iterator, added_breaks=['in']) if tok != 'in' or middle is None: - #if middle is None: - # level -= 1 - #else: - #middle.parent = self.scope - debug.warning('list comprehension formatting @%s' % - start_pos[0]) + debug.warning('list comprehension middle @%s' % str(start_pos)) return None, tok in_clause, tok = parse_stmt_or_arr(token_iterator) - """ - if tok not in b or in_clause is None: - #middle.parent = self.scope - if in_clause is None: - self._gen.push_last_back() - #else: - # in_clause.parent = self.scope - # in_clause.parent = self.scope - debug.warning('list comprehension in_clause %s@%s' - % (repr(tok), start_pos[0])) + if in_clause is None: + debug.warning('list comprehension in @%s' % str(start_pos)) return None, tok - """ - """ - other_level = 0 - for i, tok in enumerate(reversed(token_list)): - if not isinstance(tok, (Name, ListComprehension)): - tok = tok[1] - if tok in closing_brackets: - other_level -= 1 - elif tok in brackets.keys(): - other_level += 1 - if other_level > 0: - break - else: - # could not detect brackets -> nested list comp - i = 0 -""" - #token_list, toks = token_list[:-i], token_list[-i:-1] - - - if middle is None or in_clause is None: - return None, tok return ListComprehension(st, middle, in_clause, self), tok # initializations @@ -1136,7 +1102,12 @@ class Call(Simple): if self.type == Call.NAME: s = self.name.get_code() else: - s = '' if self.name is None else repr(self.name) + if isinstance(self.name, str) and "'" not in self.name: + # This is a very rough spot, because of repr not supporting + # unicode signs, see `test_unicode_script`. + s = "'%s'" % unicode(self.name, 'UTF-8') + else: + s = '' if self.name is None else repr(self.name) if self.execution is not None: s += self.execution.get_code() if self.next is not None: diff --git a/test/regression.py b/test/regression.py index fdf431f9..c3a60b3f 100755 --- a/test/regression.py +++ b/test/regression.py @@ -240,7 +240,8 @@ class TestRegression(TestBase): s = utf8("author='öä'; author") completions = self.complete(s) - assert type(completions[0].description) is unicode + x = completions[0].description + assert type(x) is unicode s = utf8("#-*- coding: iso-8859-1 -*-\nauthor='öä'; author") s = s.encode('latin-1') From 6ab10b6fa1e0580789671f1e62c986ba34f624e4 Mon Sep 17 00:00:00 2001 From: David Halter Date: Sat, 23 Feb 2013 01:16:02 +0430 Subject: [PATCH 82/83] pep8 --- jedi/evaluate_representation.py | 2 +- jedi/parsing_representation.py | 23 ++++++++++++++--------- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/jedi/evaluate_representation.py b/jedi/evaluate_representation.py index 197d8cc5..e294e1d1 100644 --- a/jedi/evaluate_representation.py +++ b/jedi/evaluate_representation.py @@ -589,7 +589,7 @@ class Execution(Executable): # No value: return the default values. ignore_creation = True result.append(param.get_name()) - param.is_generated=True + param.is_generated = True else: # If there is no assignment detail, that means there is # no assignment, just the result. Therefore nothing has diff --git a/jedi/parsing_representation.py b/jedi/parsing_representation.py index 2237e680..e18668e1 100644 --- a/jedi/parsing_representation.py +++ b/jedi/parsing_representation.py @@ -46,7 +46,8 @@ class Simple(Base): The super class for Scope, Import, Name and Statement. Every object in the parser tree inherits from this class. """ - __slots__ = ('parent', '_sub_module', '_start_pos', 'use_as_parent', '_end_pos') + __slots__ = ('parent', '_sub_module', '_start_pos', 'use_as_parent', + '_end_pos') def __init__(self, module, start_pos, end_pos=(None, None)): self._sub_module = module @@ -59,7 +60,8 @@ class Simple(Base): @property def start_pos(self): - return self._sub_module.line_offset + self._start_pos[0], self._start_pos[1] + return self._sub_module.line_offset + self._start_pos[0], \ + self._start_pos[1] @start_pos.setter def start_pos(self, value): @@ -69,7 +71,8 @@ class Simple(Base): def end_pos(self): if None in self._end_pos: return self._end_pos - return self._sub_module.line_offset + self._end_pos[0], self._end_pos[1] + return self._sub_module.line_offset + self._end_pos[0], \ + self._end_pos[1] @end_pos.setter def end_pos(self, value): @@ -536,7 +539,8 @@ class ForFlow(Flow): """ Used for the for loop, because there are two statement parts. """ - def __init__(self, module, inputs, start_pos, set_stmt, is_list_comp=False): + def __init__(self, module, inputs, start_pos, set_stmt, + is_list_comp=False): super(ForFlow, self).__init__(module, 'for', inputs, start_pos, set_stmt.used_vars) self.set_stmt = set_stmt @@ -617,8 +621,8 @@ class Import(Simple): return [self.alias] if len(self.namespace) > 1: o = self.namespace - n = Name(self._sub_module, [(o.names[0], o.start_pos)], o.start_pos, - o.end_pos, parent=o.parent) + n = Name(self._sub_module, [(o.names[0], o.start_pos)], + o.start_pos, o.end_pos, parent=o.parent) return [n] else: return [self.namespace] @@ -885,7 +889,8 @@ class Statement(Simple): def parse_list_comp(token_iterator, token_list, start_pos, end_pos): def parse_stmt_or_arr(token_iterator, added_breaks=()): - stmt, tok = parse_stmt(token_iterator, added_breaks=added_breaks) + stmt, tok = parse_stmt(token_iterator, + added_breaks=added_breaks) if not stmt: return None, tok if tok == ',': @@ -908,7 +913,8 @@ class Statement(Simple): st = Statement(self._sub_module, [], [], token_list, start_pos, end_pos) - middle, tok = parse_stmt_or_arr(token_iterator, added_breaks=['in']) + middle, tok = parse_stmt_or_arr(token_iterator, + added_breaks=['in']) if tok != 'in' or middle is None: debug.warning('list comprehension middle @%s' % str(start_pos)) return None, tok @@ -997,7 +1003,6 @@ class Statement(Simple): stmt = Statement(self._sub_module, [], [], result, start_pos, end_pos, self.parent) stmt._commands = result - #add_el, t = parse_stmt(enumerate(self.token_list[start:i - 1])) arr, break_tok = parse_array(token_iterator, Array.TUPLE, stmt.start_pos, stmt) result = [arr] From adc7a559cfcaa486cb76e6b9480d66af41f2255a Mon Sep 17 00:00:00 2001 From: David Halter Date: Sat, 23 Feb 2013 08:58:15 +0430 Subject: [PATCH 83/83] python3.2 compatibility --- jedi/parsing_representation.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/jedi/parsing_representation.py b/jedi/parsing_representation.py index e18668e1..08aae84b 100644 --- a/jedi/parsing_representation.py +++ b/jedi/parsing_representation.py @@ -1107,7 +1107,8 @@ class Call(Simple): if self.type == Call.NAME: s = self.name.get_code() else: - if isinstance(self.name, str) and "'" not in self.name: + if not is_py3k and isinstance(self.name, str)\ + and "'" not in self.name: # This is a very rough spot, because of repr not supporting # unicode signs, see `test_unicode_script`. s = "'%s'" % unicode(self.name, 'UTF-8')