forked from VimPlug/jedi
remove old statement parser code.
This commit is contained in:
Dave Halter
@@ -1107,231 +1107,8 @@ class Statement(Simple, DocstringMixin):
|
||||
def expression_list(self):
|
||||
"""
|
||||
Parse a statement.
|
||||
|
||||
This is not done in the main parser, because it might be slow and
|
||||
most of the statements won't need this data anyway. This is something
|
||||
'like' a lazy execution.
|
||||
|
||||
This is not really nice written, sorry for that. If you plan to replace
|
||||
it and make it nicer, that would be cool :-)
|
||||
"""
|
||||
def is_assignment(tok):
|
||||
return isinstance(tok, Operator) and tok.string.endswith('=') \
|
||||
and tok.string not in ('>=', '<=', '==', '!=')
|
||||
|
||||
def parse_array(token_iterator, array_type, start_pos, add_el=None):
|
||||
arr = Array(self._sub_module, start_pos, array_type, self)
|
||||
if add_el is not None:
|
||||
arr.add_statement(add_el)
|
||||
old_stmt = add_el
|
||||
|
||||
maybe_dict = array_type == Array.SET
|
||||
break_tok = None
|
||||
is_array = None
|
||||
while True:
|
||||
stmt, break_tok = parse_stmt(token_iterator, maybe_dict,
|
||||
break_on_assignment=bool(add_el))
|
||||
if stmt is None:
|
||||
break
|
||||
else:
|
||||
if break_tok == ',':
|
||||
is_array = True
|
||||
arr.add_statement(stmt, is_key=maybe_dict and break_tok == ':')
|
||||
if break_tok in closing_brackets \
|
||||
or is_assignment(break_tok):
|
||||
break
|
||||
old_stmt = stmt
|
||||
if arr.type == Array.TUPLE and len(arr) == 1 and not is_array:
|
||||
arr.type = Array.NOARRAY
|
||||
if not arr.values and maybe_dict:
|
||||
# this is a really special case - empty brackets {} are
|
||||
# always dictionaries and not sets.
|
||||
arr.type = Array.DICT
|
||||
|
||||
try:
|
||||
arr.end_pos = (break_tok or stmt or old_stmt).end_pos
|
||||
except UnboundLocalError:
|
||||
# In case of something like `(def`
|
||||
arr.end_pos = start_pos[0], start_pos[1] + 1
|
||||
return arr, break_tok
|
||||
|
||||
def parse_stmt(token_iterator, maybe_dict=False, added_breaks=(),
|
||||
break_on_assignment=False, stmt_class=ArrayStmt,
|
||||
allow_comma=False):
|
||||
token_list = []
|
||||
level = 0
|
||||
first = True
|
||||
end_pos = None, None
|
||||
tok = None
|
||||
for tok in token_iterator:
|
||||
end_pos = tok.end_pos
|
||||
if first:
|
||||
start_pos = tok.start_pos
|
||||
first = False
|
||||
|
||||
if isinstance(tok, Base):
|
||||
# The token is a Name, which has already been parsed.
|
||||
if not level:
|
||||
if isinstance(tok, ListComprehension):
|
||||
# it's not possible to set it earlier
|
||||
tok.parent = self
|
||||
elif tok == 'lambda':
|
||||
lambd, tok = parse_lambda(token_iterator)
|
||||
if lambd is not None:
|
||||
token_list.append(lambd)
|
||||
elif tok == 'for':
|
||||
list_comp, tok = parse_list_comp(token_iterator, token_list,
|
||||
start_pos, tok.end_pos)
|
||||
if list_comp is not None:
|
||||
token_list = [list_comp]
|
||||
|
||||
if tok in closing_brackets:
|
||||
level -= 1
|
||||
elif tok in brackets.keys():
|
||||
level += 1
|
||||
|
||||
if level == -1 or level == 0 and (
|
||||
tok == ',' and not allow_comma
|
||||
or tok in added_breaks
|
||||
or maybe_dict and tok == ':'
|
||||
or is_assignment(tok) and break_on_assignment):
|
||||
end_pos = end_pos[0], end_pos[1] - 1
|
||||
break
|
||||
|
||||
if tok is not None: # Can be None, because of lambda/for.
|
||||
token_list.append(tok)
|
||||
|
||||
if not token_list:
|
||||
return None, tok
|
||||
|
||||
statement = stmt_class(self._sub_module, token_list, start_pos,
|
||||
end_pos, self.parent, set_name_parents=False)
|
||||
return statement, tok
|
||||
|
||||
def parse_lambda(token_iterator):
|
||||
params = []
|
||||
start_pos = self.start_pos
|
||||
while True:
|
||||
param, tok = parse_stmt(token_iterator, added_breaks=[':'],
|
||||
stmt_class=Param)
|
||||
if param is None:
|
||||
break
|
||||
params.append(param)
|
||||
if tok == ':':
|
||||
break
|
||||
if tok != ':':
|
||||
return None, tok
|
||||
|
||||
# Since Lambda is a Function scope, it needs Scope parents.
|
||||
lambd = Lambda(self._sub_module, params, start_pos, self)
|
||||
|
||||
ret, tok = parse_stmt(token_iterator)
|
||||
if ret is not None:
|
||||
ret.parent = lambd
|
||||
lambd.statements.append(ret)
|
||||
lambd.returns.append(ret)
|
||||
lambd.end_pos = self.end_pos
|
||||
return lambd, tok
|
||||
|
||||
def parse_list_comp(token_iterator, token_list, start_pos, end_pos):
|
||||
def parse_stmt_or_arr(token_iterator, added_breaks=(),
|
||||
names_are_set_vars=False):
|
||||
stmt, tok = parse_stmt(token_iterator, allow_comma=True,
|
||||
added_breaks=added_breaks)
|
||||
|
||||
if stmt is not None:
|
||||
stmt._names_are_set_vars = names_are_set_vars
|
||||
return stmt, tok
|
||||
|
||||
st = ArrayStmt(self._sub_module, token_list, start_pos,
|
||||
end_pos, set_name_parents=False)
|
||||
|
||||
middle, tok = parse_stmt_or_arr(token_iterator, ['in'], True)
|
||||
if tok != 'in' or middle is None:
|
||||
debug.warning('list comprehension middle %s@%s', tok, start_pos)
|
||||
return None, tok
|
||||
|
||||
in_clause, tok = parse_stmt_or_arr(token_iterator)
|
||||
if in_clause is None:
|
||||
debug.warning('list comprehension in @%s', start_pos)
|
||||
return None, tok
|
||||
|
||||
return ListComprehension(self._sub_module, st, middle, in_clause, self), tok
|
||||
|
||||
# initializations
|
||||
result = []
|
||||
is_chain = False
|
||||
brackets = {'(': Array.TUPLE, '[': Array.LIST, '{': Array.SET}
|
||||
closing_brackets = ')', '}', ']'
|
||||
|
||||
token_iterator = iter(self._token_list)
|
||||
for tok in token_iterator:
|
||||
if isinstance(tok, tokenize.Token):
|
||||
token_type = tok.type
|
||||
tok_str = tok.string
|
||||
if tok_str == 'as': # just ignore as, because it sets values
|
||||
next(token_iterator, None)
|
||||
continue
|
||||
else:
|
||||
# The token is a Name, which has already been parsed
|
||||
tok_str = tok
|
||||
token_type = None
|
||||
|
||||
if is_assignment(tok):
|
||||
# This means, there is an assignment here.
|
||||
# Add assignments, which can be more than one
|
||||
self._assignment_details.append((result, tok))
|
||||
result = []
|
||||
is_chain = False
|
||||
continue
|
||||
|
||||
if tok_str == 'lambda':
|
||||
lambd, tok_str = parse_lambda(token_iterator)
|
||||
if lambd is not None:
|
||||
result.append(lambd)
|
||||
if tok_str != ',':
|
||||
continue
|
||||
|
||||
is_literal = token_type in (tokenize.STRING, tokenize.NUMBER)
|
||||
if is_literal or isinstance(tok, Name):
|
||||
cls = Literal if is_literal else Call
|
||||
|
||||
call = cls(self._sub_module, tok_str, tok.start_pos, tok.end_pos, self)
|
||||
if is_chain:
|
||||
result[-1].set_next(call)
|
||||
else:
|
||||
result.append(call)
|
||||
is_chain = False
|
||||
elif tok_str in brackets.keys():
|
||||
arr, is_ass = parse_array(
|
||||
token_iterator, brackets[tok.string], tok.start_pos
|
||||
)
|
||||
if result and isinstance(result[-1], StatementElement):
|
||||
result[-1].set_next(arr)
|
||||
else:
|
||||
arr.parent = self
|
||||
result.append(arr)
|
||||
elif tok_str == '.':
|
||||
if result and isinstance(result[-1], StatementElement):
|
||||
is_chain = True
|
||||
elif tok_str == ',' and result: # implies a tuple
|
||||
# expression is now an array not a statement anymore
|
||||
stmt = ArrayStmt(self._sub_module, result, result[0].start_pos,
|
||||
tok.end_pos, self.parent, set_name_parents=False)
|
||||
stmt._expression_list = result
|
||||
arr, break_tok = parse_array(token_iterator, Array.TUPLE,
|
||||
stmt.start_pos, stmt)
|
||||
result = [arr]
|
||||
if is_assignment(break_tok):
|
||||
self._assignment_details.append((result, break_tok))
|
||||
result = []
|
||||
is_chain = False
|
||||
else:
|
||||
# comments, strange tokens (like */**), error tokens to
|
||||
# reproduce the string correctly.
|
||||
is_chain = False
|
||||
result.append(tok)
|
||||
return result
|
||||
return self.children
|
||||
|
||||
def set_expression_list(self, lst):
|
||||
"""It's necessary for some "hacks" to change the expression_list."""
|
||||
|
||||
Reference in New Issue
Block a user