VimPlug/jedi-fork
1
0
Fork 0
forked from VimPlug/jedi
Code Pull Requests Activity

basic listcomprehension/lambda move

Browse Source
This commit is contained in:
David Halter
2013-02-22 23:02:44 +04:30
parent b0c3fd5439
commit 2fda713118
3 changed files with 138 additions and 97 deletions
Download Patch File Download Diff File Expand all files Collapse all files

90
jedi/parsing.py
View File

@@ -297,7 +297,7 @@ class Parser(object):
return scope return scope
def _parse_statement(self, pre_used_token=None, added_breaks=None, def _parse_statement(self, pre_used_token=None, added_breaks=None,
stmt_class=pr.Statement, list_comp=False): stmt_class=pr.Statement):
""" """
Parses statements like: Parses statements like:
@@ -355,86 +355,16 @@ class Parser(object):
set_vars.append(n) set_vars.append(n)
tok_list.append(n) tok_list.append(n)
continue continue
elif tok == 'lambda': elif tok in ['lambda', 'for', 'in']:
params = [] pass # don't parse these keywords, parse later in stmt.
start_pos = self.start_pos
while tok != ':':
param, tok = self._parse_statement(
added_breaks=[':', ','], stmt_class=pr.Param)
if param is None:
break
params.append(param)
if tok != ':':
continue
lambd = pr.Lambda(self.module, params, start_pos)
ret, tok = self._parse_statement(added_breaks=[','])
if ret is not None:
ret.parent = lambd
lambd.returns.append(ret)
lambd.parent = self.scope
lambd.end_pos = self.end_pos
tok_list[-1] = lambd
continue
elif token_type == tokenize.NAME: elif token_type == tokenize.NAME:
if tok == 'for': n, token_type, tok = self._parse_dot_name(self.current)
# list comprehensions! # removed last entry, because we add Name
middle, tok = self._parse_statement( tok_list.pop()
added_breaks=['in']) if n:
if tok != 'in' or middle is None: tok_list.append(n)
if middle is None: used_vars.append(n)
level -= 1 continue
else:
middle.parent = self.scope
debug.warning('list comprehension formatting @%s' %
self.start_pos[0])
continue
b = [')', ']']
in_clause, tok = self._parse_statement(added_breaks=b,
list_comp=True)
if tok not in b or in_clause is None:
middle.parent = self.scope
if in_clause is None:
self._gen.push_last_back()
else:
in_clause.parent = self.scope
in_clause.parent = self.scope
debug.warning('list comprehension in_clause %s@%s'
% (repr(tok), self.start_pos[0]))
continue
other_level = 0
for i, tok in enumerate(reversed(tok_list)):
if not isinstance(tok, (pr.Name,
pr.ListComprehension)):
tok = tok[1]
if tok in closing_brackets:
other_level -= 1
elif tok in opening_brackets:
other_level += 1
if other_level > 0:
break
else:
# could not detect brackets -> nested list comp
i = 0
tok_list, toks = tok_list[:-i], tok_list[-i:-1]
st = pr.Statement(self.module, [], [],
toks, first_pos, self.end_pos)
tok = pr.ListComprehension(st, middle, in_clause,
self.scope)
tok_list.append(tok)
continue
else:
n, token_type, tok = self._parse_dot_name(self.current)
# removed last entry, because we add Name
tok_list.pop()
if n:
tok_list.append(n)
used_vars.append(n)
continue
elif tok.endswith('=') and tok not in ['>=', '<=', '==', '!=']: elif tok.endswith('=') and tok not in ['>=', '<=', '==', '!=']:
# there has been an assignement -> change vars # there has been an assignement -> change vars
if level == 0: if level == 0:

143
jedi/parsing_representation.py
View File

@@ -499,9 +499,10 @@ class Flow(Scope):
""" """
Get the names for the flow. This includes also a call to the super Get the names for the flow. This includes also a call to the super
class. class.
:param is_internal_call: defines an option for internal files to crawl\
through this class. Normally it will just call its superiors, to\ :param is_internal_call: defines an option for internal files to crawl
generate the output. through this class. Normally it will just call its superiors, to
generate the output.
""" """
if is_internal_call: if is_internal_call:
n = list(self.set_vars) n = list(self.set_vars)
@@ -521,7 +522,7 @@ class Flow(Scope):
return i return i
def set_next(self, next): def set_next(self, next):
""" Set the next element in the flow, those are else, except, etc. """ """Set the next element in the flow, those are else, except, etc."""
if self.next: if self.next:
return self.next.set_next(next) return self.next.set_next(next)
else: else:
@@ -756,7 +757,8 @@ class Statement(Simple):
return isinstance(tok, (str, unicode)) and tok.endswith('=') \ return isinstance(tok, (str, unicode)) and tok.endswith('=') \
and not tok in ['>=', '<=', '==', '!='] and not tok in ['>=', '<=', '==', '!=']
def parse_array(token_iterator, array_type, start_pos, add_el=None): def parse_array(token_iterator, array_type, start_pos, add_el=None,
added_breaks=()):
arr = Array(self._sub_module, start_pos, array_type, self) arr = Array(self._sub_module, start_pos, array_type, self)
if add_el is not None: if add_el is not None:
arr.add_statement(add_el) arr.add_statement(add_el)
@@ -765,8 +767,9 @@ class Statement(Simple):
break_tok = None break_tok = None
is_array = None is_array = None
while True: while True:
stmt, break_tok = parse_array_el(token_iterator, maybe_dict, stmt, break_tok = parse_stmt(token_iterator, maybe_dict,
break_on_assignment=bool(add_el)) break_on_assignment=bool(add_el),
added_breaks=added_breaks)
if stmt is None: if stmt is None:
break break
else: else:
@@ -774,7 +777,9 @@ class Statement(Simple):
is_array = True is_array = True
is_key = maybe_dict and break_tok == ':' is_key = maybe_dict and break_tok == ':'
arr.add_statement(stmt, is_key) arr.add_statement(stmt, is_key)
if break_tok in closing_brackets or is_assignment(break_tok): if break_tok in closing_brackets \
or break_tok in added_breaks \
or is_assignment(break_tok):
break break
if arr.type == Array.TUPLE and len(arr) == 1 and not is_array: if arr.type == Array.TUPLE and len(arr) == 1 and not is_array:
arr.type = Array.NOARRAY arr.type = Array.NOARRAY
@@ -791,12 +796,14 @@ class Statement(Simple):
else 0) else 0)
return arr, break_tok return arr, break_tok
def parse_array_el(token_iterator, maybe_dict=False, def parse_stmt(token_iterator, maybe_dict=False, added_breaks=(),
break_on_assignment=False): break_on_assignment=False, stmt_class=Statement):
token_list = [] token_list = []
used_vars = []
level = 1 level = 1
tok = None tok = None
first = True first = True
end_pos = None
for i, tok_temp in token_iterator: for i, tok_temp in token_iterator:
if isinstance(tok_temp, Base): if isinstance(tok_temp, Base):
# the token is a Name, which has already been parsed # the token is a Name, which has already been parsed
@@ -808,20 +815,34 @@ class Statement(Simple):
if isinstance(tok, ListComprehension): if isinstance(tok, ListComprehension):
# it's not possible to set it earlier # it's not possible to set it earlier
tok.parent = self tok.parent = self
if isinstance(tok, Name):
used_vars.append(tok)
else: else:
token_type, tok, start_tok_pos = tok_temp token_type, tok, start_tok_pos = tok_temp
last_end_pos = end_pos
end_pos = start_tok_pos[0], start_tok_pos[1] + len(tok) end_pos = start_tok_pos[0], start_tok_pos[1] + len(tok)
if first: if first:
first = False first = False
start_pos = start_tok_pos start_pos = start_tok_pos
if tok == 'lambda':
lambd = parse_lambda(token_iterator)
if lambd is not None:
token_list.append(lambd)
elif tok == 'for':
list_comp, tok = parse_list_comp(token_iterator,
token_list, start_pos, last_end_pos)
if list_comp is not None:
token_list = [list_comp]
if tok in closing_brackets: if tok in closing_brackets:
level -= 1 level -= 1
elif tok in brackets.keys(): elif tok in brackets.keys():
level += 1 level += 1
if level == 0 and tok in closing_brackets \ if level == 0 and tok in closing_brackets \
or level == 1 and (tok == ',' or tok in added_breaks \
or level == 1 and (tok == ','
or maybe_dict and tok == ':' or maybe_dict and tok == ':'
or is_assignment(tok) and break_on_assignment): or is_assignment(tok) and break_on_assignment):
end_pos = end_pos[0], end_pos[1] - 1 end_pos = end_pos[0], end_pos[1] - 1
@@ -831,11 +852,101 @@ class Statement(Simple):
if not token_list: if not token_list:
return None, tok return None, tok
statement = Statement(self._sub_module, [], [], statement = Statement(self._sub_module, [], [], token_list,
token_list, start_pos, end_pos) start_pos, end_pos, self.parent)
statement.parent = self.parent statement.used_vars = used_vars
return statement, tok return statement, tok
def parse_lambda(token_iterator):
params = []
start_pos = self.start_pos
tok = next(token_iterator)
while tok != ':':
param, tok = parse_stmt(token_iterator,
added_breaks=[':', ','], stmt_class=Param)
if param is None:
break
params.append(param)
if tok != ':':
return None, tok
lambd = Lambda(self.module, params, start_pos)
ret, tok = self._parse_statement(added_breaks=[','])
if ret is not None:
ret.parent = lambd
lambd.returns.append(ret)
lambd.parent = self.scope
lambd.end_pos = self.end_pos
return lambd
def parse_list_comp(token_iterator, token_list, start_pos, end_pos):
def parse_stmt_or_arr(token_iterator, added_breaks=()):
stmt, tok = parse_stmt(token_iterator, added_breaks=added_breaks)
if tok == ',':
arr, tok = parse_array(token_iterator, Array.TUPLE,
stmt.start_pos, stmt,
added_breaks=added_breaks)
used_vars = []
for stmt in arr:
used_vars += stmt.used_vars
start_pos = arr.start_pos[0], arr.start_pos[1] - 1
stmt = Statement(self._sub_module, [], used_vars, [],
start_pos, arr.end_pos)
arr.parent = stmt
stmt.token_list = stmt._commands = [arr]
else:
for v in stmt.used_vars:
v.parent = stmt
return stmt, tok
st = Statement(self._sub_module, [], [], token_list, start_pos,
end_pos)
middle, tok = parse_stmt_or_arr(token_iterator, added_breaks=['in'])
if tok != 'in' or middle is None:
#if middle is None:
# level -= 1
#else:
#middle.parent = self.scope
debug.warning('list comprehension formatting @%s' %
start_pos[0])
return None, tok
in_clause, tok = parse_stmt_or_arr(token_iterator)
"""
if tok not in b or in_clause is None:
#middle.parent = self.scope
if in_clause is None:
self._gen.push_last_back()
#else:
# in_clause.parent = self.scope
# in_clause.parent = self.scope
debug.warning('list comprehension in_clause %s@%s'
% (repr(tok), start_pos[0]))
return None, tok
"""
"""
other_level = 0
for i, tok in enumerate(reversed(token_list)):
if not isinstance(tok, (Name, ListComprehension)):
tok = tok[1]
if tok in closing_brackets:
other_level -= 1
elif tok in brackets.keys():
other_level += 1
if other_level > 0:
break
else:
# could not detect brackets -> nested list comp
i = 0
"""
#token_list, toks = token_list[:-i], token_list[-i:-1]
return ListComprehension(st, middle, in_clause, self), tok
# initializations # initializations
result = [] result = []
is_chain = False is_chain = False
@@ -860,7 +971,7 @@ class Statement(Simple):
is_chain = False is_chain = False
start = i + 1 start = i + 1
continue continue
elif tok == 'as': # just ignore as elif tok == 'as': # just ignore as, because it sets values
next(token_iterator, None) next(token_iterator, None)
continue continue
@@ -893,7 +1004,7 @@ class Statement(Simple):
is_chain = True is_chain = True
elif tok == ',': # implies a tuple elif tok == ',': # implies a tuple
# rewrite `result`, because now the whole thing is a tuple # rewrite `result`, because now the whole thing is a tuple
add_el, t = parse_array_el(enumerate(self.token_list[start:i])) add_el, t = parse_stmt(enumerate(self.token_list[start:i]))
arr, break_tok = parse_array(token_iterator, Array.TUPLE, arr, break_tok = parse_array(token_iterator, Array.TUPLE,
add_el.start_pos, add_el) add_el.start_pos, add_el)
result = [arr] result = [arr]

2
jedi/recursion.py
View File

@@ -35,7 +35,7 @@ class RecursionDecorator(object):
def push_stmt(self, stmt): def push_stmt(self, stmt):
self.current = RecursionNode(stmt, self.current) self.current = RecursionNode(stmt, self.current)
check = self._check_recursion() check = self._check_recursion()
if check: if check:# TODO remove False!!!!
debug.warning('catched stmt recursion: %s against %s @%s' debug.warning('catched stmt recursion: %s against %s @%s'
% (stmt, check.stmt, stmt.start_pos)) % (stmt, check.stmt, stmt.start_pos))
self.pop_stmt() self.pop_stmt()