VimPlug/jedi-fork
1
0
Fork 0
forked from VimPlug/jedi
Code Pull Requests Activity

remove old statement parser code.

Browse Source
This commit is contained in:
Dave Halter
2014-10-09 11:20:49 +02:00
parent 140320143a
commit a0092c2653
1 changed files with 1 additions and 224 deletions
Download Patch File Download Diff File Expand all files Collapse all files

225
jedi/parser/representation.py
View File

@@ -1107,231 +1107,8 @@ class Statement(Simple, DocstringMixin):
def expression_list(self): def expression_list(self):
""" """
Parse a statement. Parse a statement.
This is not done in the main parser, because it might be slow and
most of the statements won't need this data anyway. This is something
'like' a lazy execution.
This is not really nice written, sorry for that. If you plan to replace
it and make it nicer, that would be cool :-)
""" """
def is_assignment(tok): return self.children
return isinstance(tok, Operator) and tok.string.endswith('=') \
and tok.string not in ('>=', '<=', '==', '!=')
def parse_array(token_iterator, array_type, start_pos, add_el=None):
arr = Array(self._sub_module, start_pos, array_type, self)
if add_el is not None:
arr.add_statement(add_el)
old_stmt = add_el
maybe_dict = array_type == Array.SET
break_tok = None
is_array = None
while True:
stmt, break_tok = parse_stmt(token_iterator, maybe_dict,
break_on_assignment=bool(add_el))
if stmt is None:
break
else:
if break_tok == ',':
is_array = True
arr.add_statement(stmt, is_key=maybe_dict and break_tok == ':')
if break_tok in closing_brackets \
or is_assignment(break_tok):
break
old_stmt = stmt
if arr.type == Array.TUPLE and len(arr) == 1 and not is_array:
arr.type = Array.NOARRAY
if not arr.values and maybe_dict:
# this is a really special case - empty brackets {} are
# always dictionaries and not sets.
arr.type = Array.DICT
try:
arr.end_pos = (break_tok or stmt or old_stmt).end_pos
except UnboundLocalError:
# In case of something like `(def`
arr.end_pos = start_pos[0], start_pos[1] + 1
return arr, break_tok
def parse_stmt(token_iterator, maybe_dict=False, added_breaks=(),
break_on_assignment=False, stmt_class=ArrayStmt,
allow_comma=False):
token_list = []
level = 0
first = True
end_pos = None, None
tok = None
for tok in token_iterator:
end_pos = tok.end_pos
if first:
start_pos = tok.start_pos
first = False
if isinstance(tok, Base):
# The token is a Name, which has already been parsed.
if not level:
if isinstance(tok, ListComprehension):
# it's not possible to set it earlier
tok.parent = self
elif tok == 'lambda':
lambd, tok = parse_lambda(token_iterator)
if lambd is not None:
token_list.append(lambd)
elif tok == 'for':
list_comp, tok = parse_list_comp(token_iterator, token_list,
start_pos, tok.end_pos)
if list_comp is not None:
token_list = [list_comp]
if tok in closing_brackets:
level -= 1
elif tok in brackets.keys():
level += 1
if level == -1 or level == 0 and (
tok == ',' and not allow_comma
or tok in added_breaks
or maybe_dict and tok == ':'
or is_assignment(tok) and break_on_assignment):
end_pos = end_pos[0], end_pos[1] - 1
break
if tok is not None: # Can be None, because of lambda/for.
token_list.append(tok)
if not token_list:
return None, tok
statement = stmt_class(self._sub_module, token_list, start_pos,
end_pos, self.parent, set_name_parents=False)
return statement, tok
def parse_lambda(token_iterator):
params = []
start_pos = self.start_pos
while True:
param, tok = parse_stmt(token_iterator, added_breaks=[':'],
stmt_class=Param)
if param is None:
break
params.append(param)
if tok == ':':
break
if tok != ':':
return None, tok
# Since Lambda is a Function scope, it needs Scope parents.
lambd = Lambda(self._sub_module, params, start_pos, self)
ret, tok = parse_stmt(token_iterator)
if ret is not None:
ret.parent = lambd
lambd.statements.append(ret)
lambd.returns.append(ret)
lambd.end_pos = self.end_pos
return lambd, tok
def parse_list_comp(token_iterator, token_list, start_pos, end_pos):
def parse_stmt_or_arr(token_iterator, added_breaks=(),
names_are_set_vars=False):
stmt, tok = parse_stmt(token_iterator, allow_comma=True,
added_breaks=added_breaks)
if stmt is not None:
stmt._names_are_set_vars = names_are_set_vars
return stmt, tok
st = ArrayStmt(self._sub_module, token_list, start_pos,
end_pos, set_name_parents=False)
middle, tok = parse_stmt_or_arr(token_iterator, ['in'], True)
if tok != 'in' or middle is None:
debug.warning('list comprehension middle %s@%s', tok, start_pos)
return None, tok
in_clause, tok = parse_stmt_or_arr(token_iterator)
if in_clause is None:
debug.warning('list comprehension in @%s', start_pos)
return None, tok
return ListComprehension(self._sub_module, st, middle, in_clause, self), tok
# initializations
result = []
is_chain = False
brackets = {'(': Array.TUPLE, '[': Array.LIST, '{': Array.SET}
closing_brackets = ')', '}', ']'
token_iterator = iter(self._token_list)
for tok in token_iterator:
if isinstance(tok, tokenize.Token):
token_type = tok.type
tok_str = tok.string
if tok_str == 'as': # just ignore as, because it sets values
next(token_iterator, None)
continue
else:
# The token is a Name, which has already been parsed
tok_str = tok
token_type = None
if is_assignment(tok):
# This means, there is an assignment here.
# Add assignments, which can be more than one
self._assignment_details.append((result, tok))
result = []
is_chain = False
continue
if tok_str == 'lambda':
lambd, tok_str = parse_lambda(token_iterator)
if lambd is not None:
result.append(lambd)
if tok_str != ',':
continue
is_literal = token_type in (tokenize.STRING, tokenize.NUMBER)
if is_literal or isinstance(tok, Name):
cls = Literal if is_literal else Call
call = cls(self._sub_module, tok_str, tok.start_pos, tok.end_pos, self)
if is_chain:
result[-1].set_next(call)
else:
result.append(call)
is_chain = False
elif tok_str in brackets.keys():
arr, is_ass = parse_array(
token_iterator, brackets[tok.string], tok.start_pos
)
if result and isinstance(result[-1], StatementElement):
result[-1].set_next(arr)
else:
arr.parent = self
result.append(arr)
elif tok_str == '.':
if result and isinstance(result[-1], StatementElement):
is_chain = True
elif tok_str == ',' and result: # implies a tuple
# expression is now an array not a statement anymore
stmt = ArrayStmt(self._sub_module, result, result[0].start_pos,
tok.end_pos, self.parent, set_name_parents=False)
stmt._expression_list = result
arr, break_tok = parse_array(token_iterator, Array.TUPLE,
stmt.start_pos, stmt)
result = [arr]
if is_assignment(break_tok):
self._assignment_details.append((result, break_tok))
result = []
is_chain = False
else:
# comments, strange tokens (like */**), error tokens to
# reproduce the string correctly.
is_chain = False
result.append(tok)
return result
def set_expression_list(self, lst): def set_expression_list(self, lst):
"""It's necessary for some "hacks" to change the expression_list.""" """It's necessary for some "hacks" to change the expression_list."""