diff --git a/jedi/parser/representation.py b/jedi/parser/representation.py index cf1feaca..a2ae8407 100644 --- a/jedi/parser/representation.py +++ b/jedi/parser/representation.py @@ -1107,231 +1107,8 @@ class Statement(Simple, DocstringMixin): def expression_list(self): """ Parse a statement. - - This is not done in the main parser, because it might be slow and - most of the statements won't need this data anyway. This is something - 'like' a lazy execution. - - This is not really nice written, sorry for that. If you plan to replace - it and make it nicer, that would be cool :-) """ - def is_assignment(tok): - return isinstance(tok, Operator) and tok.string.endswith('=') \ - and tok.string not in ('>=', '<=', '==', '!=') - - def parse_array(token_iterator, array_type, start_pos, add_el=None): - arr = Array(self._sub_module, start_pos, array_type, self) - if add_el is not None: - arr.add_statement(add_el) - old_stmt = add_el - - maybe_dict = array_type == Array.SET - break_tok = None - is_array = None - while True: - stmt, break_tok = parse_stmt(token_iterator, maybe_dict, - break_on_assignment=bool(add_el)) - if stmt is None: - break - else: - if break_tok == ',': - is_array = True - arr.add_statement(stmt, is_key=maybe_dict and break_tok == ':') - if break_tok in closing_brackets \ - or is_assignment(break_tok): - break - old_stmt = stmt - if arr.type == Array.TUPLE and len(arr) == 1 and not is_array: - arr.type = Array.NOARRAY - if not arr.values and maybe_dict: - # this is a really special case - empty brackets {} are - # always dictionaries and not sets. - arr.type = Array.DICT - - try: - arr.end_pos = (break_tok or stmt or old_stmt).end_pos - except UnboundLocalError: - # In case of something like `(def` - arr.end_pos = start_pos[0], start_pos[1] + 1 - return arr, break_tok - - def parse_stmt(token_iterator, maybe_dict=False, added_breaks=(), - break_on_assignment=False, stmt_class=ArrayStmt, - allow_comma=False): - token_list = [] - level = 0 - first = True - end_pos = None, None - tok = None - for tok in token_iterator: - end_pos = tok.end_pos - if first: - start_pos = tok.start_pos - first = False - - if isinstance(tok, Base): - # The token is a Name, which has already been parsed. - if not level: - if isinstance(tok, ListComprehension): - # it's not possible to set it earlier - tok.parent = self - elif tok == 'lambda': - lambd, tok = parse_lambda(token_iterator) - if lambd is not None: - token_list.append(lambd) - elif tok == 'for': - list_comp, tok = parse_list_comp(token_iterator, token_list, - start_pos, tok.end_pos) - if list_comp is not None: - token_list = [list_comp] - - if tok in closing_brackets: - level -= 1 - elif tok in brackets.keys(): - level += 1 - - if level == -1 or level == 0 and ( - tok == ',' and not allow_comma - or tok in added_breaks - or maybe_dict and tok == ':' - or is_assignment(tok) and break_on_assignment): - end_pos = end_pos[0], end_pos[1] - 1 - break - - if tok is not None: # Can be None, because of lambda/for. - token_list.append(tok) - - if not token_list: - return None, tok - - statement = stmt_class(self._sub_module, token_list, start_pos, - end_pos, self.parent, set_name_parents=False) - return statement, tok - - def parse_lambda(token_iterator): - params = [] - start_pos = self.start_pos - while True: - param, tok = parse_stmt(token_iterator, added_breaks=[':'], - stmt_class=Param) - if param is None: - break - params.append(param) - if tok == ':': - break - if tok != ':': - return None, tok - - # Since Lambda is a Function scope, it needs Scope parents. - lambd = Lambda(self._sub_module, params, start_pos, self) - - ret, tok = parse_stmt(token_iterator) - if ret is not None: - ret.parent = lambd - lambd.statements.append(ret) - lambd.returns.append(ret) - lambd.end_pos = self.end_pos - return lambd, tok - - def parse_list_comp(token_iterator, token_list, start_pos, end_pos): - def parse_stmt_or_arr(token_iterator, added_breaks=(), - names_are_set_vars=False): - stmt, tok = parse_stmt(token_iterator, allow_comma=True, - added_breaks=added_breaks) - - if stmt is not None: - stmt._names_are_set_vars = names_are_set_vars - return stmt, tok - - st = ArrayStmt(self._sub_module, token_list, start_pos, - end_pos, set_name_parents=False) - - middle, tok = parse_stmt_or_arr(token_iterator, ['in'], True) - if tok != 'in' or middle is None: - debug.warning('list comprehension middle %s@%s', tok, start_pos) - return None, tok - - in_clause, tok = parse_stmt_or_arr(token_iterator) - if in_clause is None: - debug.warning('list comprehension in @%s', start_pos) - return None, tok - - return ListComprehension(self._sub_module, st, middle, in_clause, self), tok - - # initializations - result = [] - is_chain = False - brackets = {'(': Array.TUPLE, '[': Array.LIST, '{': Array.SET} - closing_brackets = ')', '}', ']' - - token_iterator = iter(self._token_list) - for tok in token_iterator: - if isinstance(tok, tokenize.Token): - token_type = tok.type - tok_str = tok.string - if tok_str == 'as': # just ignore as, because it sets values - next(token_iterator, None) - continue - else: - # The token is a Name, which has already been parsed - tok_str = tok - token_type = None - - if is_assignment(tok): - # This means, there is an assignment here. - # Add assignments, which can be more than one - self._assignment_details.append((result, tok)) - result = [] - is_chain = False - continue - - if tok_str == 'lambda': - lambd, tok_str = parse_lambda(token_iterator) - if lambd is not None: - result.append(lambd) - if tok_str != ',': - continue - - is_literal = token_type in (tokenize.STRING, tokenize.NUMBER) - if is_literal or isinstance(tok, Name): - cls = Literal if is_literal else Call - - call = cls(self._sub_module, tok_str, tok.start_pos, tok.end_pos, self) - if is_chain: - result[-1].set_next(call) - else: - result.append(call) - is_chain = False - elif tok_str in brackets.keys(): - arr, is_ass = parse_array( - token_iterator, brackets[tok.string], tok.start_pos - ) - if result and isinstance(result[-1], StatementElement): - result[-1].set_next(arr) - else: - arr.parent = self - result.append(arr) - elif tok_str == '.': - if result and isinstance(result[-1], StatementElement): - is_chain = True - elif tok_str == ',' and result: # implies a tuple - # expression is now an array not a statement anymore - stmt = ArrayStmt(self._sub_module, result, result[0].start_pos, - tok.end_pos, self.parent, set_name_parents=False) - stmt._expression_list = result - arr, break_tok = parse_array(token_iterator, Array.TUPLE, - stmt.start_pos, stmt) - result = [arr] - if is_assignment(break_tok): - self._assignment_details.append((result, break_tok)) - result = [] - is_chain = False - else: - # comments, strange tokens (like */**), error tokens to - # reproduce the string correctly. - is_chain = False - result.append(tok) - return result + return self.children def set_expression_list(self, lst): """It's necessary for some "hacks" to change the expression_list."""