forked from VimPlug/jedi
indentation reduced with iterator (see previous commit)
This commit is contained in:
David Halter
343
jedi/parsing.py
343
jedi/parsing.py
@@ -1611,188 +1611,185 @@ class PyFuzzyParser(object):
|
|||||||
self.iterator = iter(self)
|
self.iterator = iter(self)
|
||||||
# This iterator stuff is not intentional. It grew historically.
|
# This iterator stuff is not intentional. It grew historically.
|
||||||
for token_type, tok in self.iterator:
|
for token_type, tok in self.iterator:
|
||||||
self.module.temp_used_names = []
|
self.module.temp_used_names = []
|
||||||
#debug.dbg('main: tok=[%s] type=[%s] indent=[%s]'\
|
#debug.dbg('main: tok=[%s] type=[%s] indent=[%s]'\
|
||||||
# % (tok, tokenize.tok_name[token_type], start_position[0]))
|
# % (tok, tokenize.tok_name[token_type], start_position[0]))
|
||||||
|
|
||||||
while token_type == tokenize.DEDENT \
|
while token_type == tokenize.DEDENT and self.scope != self.module:
|
||||||
and self.scope != self.module:
|
token_type, tok = self.next()
|
||||||
token_type, tok = self.next()
|
if self.start_pos[1] <= self.scope.start_pos[1]:
|
||||||
if self.start_pos[1] <= self.scope.start_pos[1]:
|
|
||||||
self.scope.end_pos = self.start_pos
|
|
||||||
self.scope = self.scope.parent()
|
|
||||||
|
|
||||||
# check again for unindented stuff. this is true for syntax
|
|
||||||
# errors. only check for names, because thats relevant here. If
|
|
||||||
# some docstrings are not indented, I don't care.
|
|
||||||
while self.start_pos[1] <= self.scope.start_pos[1] \
|
|
||||||
and (token_type == tokenize.NAME or tok in ['(', '['])\
|
|
||||||
and self.scope != self.module:
|
|
||||||
self.scope.end_pos = self.start_pos
|
self.scope.end_pos = self.start_pos
|
||||||
self.scope = self.scope.parent()
|
self.scope = self.scope.parent()
|
||||||
|
|
||||||
first_pos = self.start_pos
|
# check again for unindented stuff. this is true for syntax
|
||||||
if tok == 'def':
|
# errors. only check for names, because thats relevant here. If
|
||||||
func = self._parsefunction()
|
# some docstrings are not indented, I don't care.
|
||||||
if func is None:
|
while self.start_pos[1] <= self.scope.start_pos[1] \
|
||||||
debug.warning("function: syntax error@%s" %
|
and (token_type == tokenize.NAME or tok in ['(', '['])\
|
||||||
self.start_pos[0])
|
and self.scope != self.module:
|
||||||
continue
|
self.scope.end_pos = self.start_pos
|
||||||
self.freshscope = True
|
self.scope = self.scope.parent()
|
||||||
self.scope = self.scope.add_scope(func, decorators)
|
|
||||||
decorators = []
|
first_pos = self.start_pos
|
||||||
elif tok == 'class':
|
if tok == 'def':
|
||||||
cls = self._parseclass()
|
func = self._parsefunction()
|
||||||
if cls is None:
|
if func is None:
|
||||||
debug.warning("class: syntax error@%s" %
|
debug.warning("function: syntax error@%s" %
|
||||||
self.start_pos[0])
|
self.start_pos[0])
|
||||||
continue
|
continue
|
||||||
self.freshscope = True
|
self.freshscope = True
|
||||||
self.scope = self.scope.add_scope(cls, decorators)
|
self.scope = self.scope.add_scope(func, decorators)
|
||||||
decorators = []
|
decorators = []
|
||||||
# import stuff
|
elif tok == 'class':
|
||||||
elif tok == 'import':
|
cls = self._parseclass()
|
||||||
imports = self._parseimportlist()
|
if cls is None:
|
||||||
for m, alias, defunct in imports:
|
debug.warning("class: syntax error@%s" % self.start_pos[0])
|
||||||
i = Import(first_pos, self.end_pos, m, alias,
|
continue
|
||||||
defunct=defunct)
|
self.freshscope = True
|
||||||
self._check_user_stmt(i)
|
self.scope = self.scope.add_scope(cls, decorators)
|
||||||
self.scope.add_import(i)
|
decorators = []
|
||||||
if not imports:
|
# import stuff
|
||||||
i = Import(first_pos, self.end_pos, None, defunct=True)
|
elif tok == 'import':
|
||||||
self._check_user_stmt(i)
|
imports = self._parseimportlist()
|
||||||
self.freshscope = False
|
for m, alias, defunct in imports:
|
||||||
elif tok == 'from':
|
i = Import(first_pos, self.end_pos, m, alias,
|
||||||
defunct = False
|
defunct=defunct)
|
||||||
# take care for relative imports
|
self._check_user_stmt(i)
|
||||||
relative_count = 0
|
self.scope.add_import(i)
|
||||||
while 1:
|
if not imports:
|
||||||
token_type, tok = self.next()
|
i = Import(first_pos, self.end_pos, None, defunct=True)
|
||||||
if tok != '.':
|
self._check_user_stmt(i)
|
||||||
break
|
self.freshscope = False
|
||||||
relative_count += 1
|
elif tok == 'from':
|
||||||
# the from import
|
defunct = False
|
||||||
mod, token_type, tok = self._parsedotname(self.current)
|
# take care for relative imports
|
||||||
if str(mod) == 'import' and relative_count:
|
relative_count = 0
|
||||||
|
while 1:
|
||||||
|
token_type, tok = self.next()
|
||||||
|
if tok != '.':
|
||||||
|
break
|
||||||
|
relative_count += 1
|
||||||
|
# the from import
|
||||||
|
mod, token_type, tok = self._parsedotname(self.current)
|
||||||
|
if str(mod) == 'import' and relative_count:
|
||||||
|
self.gen.push_back(self._current_full)
|
||||||
|
tok = 'import'
|
||||||
|
mod = None
|
||||||
|
if not mod and not relative_count or tok != "import":
|
||||||
|
debug.warning("from: syntax error@%s" % self.start_pos[0])
|
||||||
|
defunct = True
|
||||||
|
if tok != 'import':
|
||||||
self.gen.push_back(self._current_full)
|
self.gen.push_back(self._current_full)
|
||||||
tok = 'import'
|
names = self._parseimportlist()
|
||||||
mod = None
|
for name, alias, defunct2 in names:
|
||||||
if not mod and not relative_count or tok != "import":
|
star = name is not None and name.names[0] == '*'
|
||||||
debug.warning("from: syntax error@%s" %
|
if star:
|
||||||
self.start_pos[0])
|
name = None
|
||||||
defunct = True
|
i = Import(first_pos, self.end_pos, name, alias, mod,
|
||||||
if tok != 'import':
|
star, relative_count, defunct=defunct or defunct2)
|
||||||
self.gen.push_back(self._current_full)
|
self._check_user_stmt(i)
|
||||||
names = self._parseimportlist()
|
self.scope.add_import(i)
|
||||||
for name, alias, defunct2 in names:
|
self.freshscope = False
|
||||||
star = name is not None and name.names[0] == '*'
|
#loops
|
||||||
if star:
|
elif tok == 'for':
|
||||||
name = None
|
set_stmt, tok = self._parse_statement(added_breaks=['in'])
|
||||||
i = Import(first_pos, self.end_pos, name, alias, mod,
|
if tok == 'in':
|
||||||
star, relative_count, defunct=defunct or defunct2)
|
statement, tok = self._parse_statement()
|
||||||
self._check_user_stmt(i)
|
if tok == ':':
|
||||||
self.scope.add_import(i)
|
s = [] if statement is None else [statement]
|
||||||
self.freshscope = False
|
f = ForFlow(s, first_pos, set_stmt)
|
||||||
#loops
|
self.scope = self.scope.add_statement(f)
|
||||||
elif tok == 'for':
|
|
||||||
set_stmt, tok = self._parse_statement(added_breaks=['in'])
|
|
||||||
if tok == 'in':
|
|
||||||
statement, tok = self._parse_statement()
|
|
||||||
if tok == ':':
|
|
||||||
s = [] if statement is None else [statement]
|
|
||||||
f = ForFlow(s, first_pos, set_stmt)
|
|
||||||
self.scope = self.scope.add_statement(f)
|
|
||||||
else:
|
|
||||||
debug.warning('syntax err, for flow started @%s',
|
|
||||||
self.start_pos[0])
|
|
||||||
if statement is not None:
|
|
||||||
statement.parent = weakref.ref(self.scope)
|
|
||||||
if set_stmt is not None:
|
|
||||||
set_stmt.parent = weakref.ref(self.scope)
|
|
||||||
else:
|
else:
|
||||||
debug.warning('syntax err, for flow incomplete @%s',
|
debug.warning('syntax err, for flow started @%s',
|
||||||
self.start_pos[0])
|
self.start_pos[0])
|
||||||
|
if statement is not None:
|
||||||
|
statement.parent = weakref.ref(self.scope)
|
||||||
if set_stmt is not None:
|
if set_stmt is not None:
|
||||||
set_stmt.parent = weakref.ref(self.scope)
|
set_stmt.parent = weakref.ref(self.scope)
|
||||||
|
|
||||||
elif tok in ['if', 'while', 'try', 'with'] + extended_flow:
|
|
||||||
added_breaks = []
|
|
||||||
command = tok
|
|
||||||
if command in ['except', 'with']:
|
|
||||||
added_breaks.append(',')
|
|
||||||
# multiple statements because of with
|
|
||||||
inits = []
|
|
||||||
first = True
|
|
||||||
while first or command == 'with' \
|
|
||||||
and tok not in [':', '\n']:
|
|
||||||
statement, tok = \
|
|
||||||
self._parse_statement(added_breaks=added_breaks)
|
|
||||||
if command == 'except' and tok in added_breaks:
|
|
||||||
# the except statement defines a var
|
|
||||||
# this is only true for python 2
|
|
||||||
n, token_type, tok = self._parsedotname()
|
|
||||||
if n:
|
|
||||||
statement.set_vars.append(n)
|
|
||||||
statement.code += ',' + n.get_code()
|
|
||||||
if statement:
|
|
||||||
inits.append(statement)
|
|
||||||
first = False
|
|
||||||
|
|
||||||
if tok == ':':
|
|
||||||
f = Flow(command, inits, first_pos)
|
|
||||||
if command in extended_flow:
|
|
||||||
# the last statement has to be another part of
|
|
||||||
# the flow statement, because a dedent releases the
|
|
||||||
# main scope, so just take the last statement.
|
|
||||||
try:
|
|
||||||
s = self.scope.statements[-1].set_next(f)
|
|
||||||
except (AttributeError, IndexError):
|
|
||||||
# If set_next doesn't exist, just add it.
|
|
||||||
s = self.scope.add_statement(f)
|
|
||||||
else:
|
|
||||||
s = self.scope.add_statement(f)
|
|
||||||
self.scope = s
|
|
||||||
else:
|
|
||||||
for i in inits:
|
|
||||||
i.parent = weakref.ref(self.scope)
|
|
||||||
debug.warning('syntax err, flow started @%s',
|
|
||||||
self.start_pos[0])
|
|
||||||
# globals
|
|
||||||
elif tok == 'global':
|
|
||||||
stmt, tok = self._parse_statement(self.current)
|
|
||||||
if stmt:
|
|
||||||
self.scope.add_statement(stmt)
|
|
||||||
for name in stmt.used_vars:
|
|
||||||
# add the global to the top, because there it is
|
|
||||||
# important.
|
|
||||||
self.module.add_global(name)
|
|
||||||
# decorator
|
|
||||||
elif tok == '@':
|
|
||||||
stmt, tok = self._parse_statement()
|
|
||||||
decorators.append(stmt)
|
|
||||||
elif tok == 'pass':
|
|
||||||
continue
|
|
||||||
elif tok == 'assert':
|
|
||||||
stmt, tok = self._parse_statement()
|
|
||||||
stmt.parent = weakref.ref(self.scope)
|
|
||||||
self.scope.asserts.append(stmt)
|
|
||||||
# default
|
|
||||||
elif token_type in [tokenize.NAME, tokenize.STRING,
|
|
||||||
tokenize.NUMBER] \
|
|
||||||
or tok in statement_toks:
|
|
||||||
# this is the main part - a name can be a function or a
|
|
||||||
# normal var, which can follow anything. but this is done
|
|
||||||
# by the statement parser.
|
|
||||||
stmt, tok = self._parse_statement(self.current)
|
|
||||||
if stmt:
|
|
||||||
self.scope.add_statement(stmt)
|
|
||||||
self.freshscope = False
|
|
||||||
else:
|
else:
|
||||||
if token_type not in [tokenize.COMMENT, tokenize.INDENT,
|
debug.warning('syntax err, for flow incomplete @%s',
|
||||||
tokenize.NEWLINE, tokenize.NL,
|
self.start_pos[0])
|
||||||
tokenize.ENDMARKER]:
|
if set_stmt is not None:
|
||||||
debug.warning('token not classified', tok, token_type,
|
set_stmt.parent = weakref.ref(self.scope)
|
||||||
self.start_pos[0])
|
|
||||||
|
elif tok in ['if', 'while', 'try', 'with'] + extended_flow:
|
||||||
|
added_breaks = []
|
||||||
|
command = tok
|
||||||
|
if command in ['except', 'with']:
|
||||||
|
added_breaks.append(',')
|
||||||
|
# multiple statements because of with
|
||||||
|
inits = []
|
||||||
|
first = True
|
||||||
|
while first or command == 'with' \
|
||||||
|
and tok not in [':', '\n']:
|
||||||
|
statement, tok = \
|
||||||
|
self._parse_statement(added_breaks=added_breaks)
|
||||||
|
if command == 'except' and tok in added_breaks:
|
||||||
|
# the except statement defines a var
|
||||||
|
# this is only true for python 2
|
||||||
|
n, token_type, tok = self._parsedotname()
|
||||||
|
if n:
|
||||||
|
statement.set_vars.append(n)
|
||||||
|
statement.code += ',' + n.get_code()
|
||||||
|
if statement:
|
||||||
|
inits.append(statement)
|
||||||
|
first = False
|
||||||
|
|
||||||
|
if tok == ':':
|
||||||
|
f = Flow(command, inits, first_pos)
|
||||||
|
if command in extended_flow:
|
||||||
|
# the last statement has to be another part of
|
||||||
|
# the flow statement, because a dedent releases the
|
||||||
|
# main scope, so just take the last statement.
|
||||||
|
try:
|
||||||
|
s = self.scope.statements[-1].set_next(f)
|
||||||
|
except (AttributeError, IndexError):
|
||||||
|
# If set_next doesn't exist, just add it.
|
||||||
|
s = self.scope.add_statement(f)
|
||||||
|
else:
|
||||||
|
s = self.scope.add_statement(f)
|
||||||
|
self.scope = s
|
||||||
|
else:
|
||||||
|
for i in inits:
|
||||||
|
i.parent = weakref.ref(self.scope)
|
||||||
|
debug.warning('syntax err, flow started @%s',
|
||||||
|
self.start_pos[0])
|
||||||
|
# globals
|
||||||
|
elif tok == 'global':
|
||||||
|
stmt, tok = self._parse_statement(self.current)
|
||||||
|
if stmt:
|
||||||
|
self.scope.add_statement(stmt)
|
||||||
|
for name in stmt.used_vars:
|
||||||
|
# add the global to the top, because there it is
|
||||||
|
# important.
|
||||||
|
self.module.add_global(name)
|
||||||
|
# decorator
|
||||||
|
elif tok == '@':
|
||||||
|
stmt, tok = self._parse_statement()
|
||||||
|
decorators.append(stmt)
|
||||||
|
elif tok == 'pass':
|
||||||
|
continue
|
||||||
|
elif tok == 'assert':
|
||||||
|
stmt, tok = self._parse_statement()
|
||||||
|
stmt.parent = weakref.ref(self.scope)
|
||||||
|
self.scope.asserts.append(stmt)
|
||||||
|
# default
|
||||||
|
elif token_type in [tokenize.NAME, tokenize.STRING,
|
||||||
|
tokenize.NUMBER] \
|
||||||
|
or tok in statement_toks:
|
||||||
|
# this is the main part - a name can be a function or a
|
||||||
|
# normal var, which can follow anything. but this is done
|
||||||
|
# by the statement parser.
|
||||||
|
stmt, tok = self._parse_statement(self.current)
|
||||||
|
if stmt:
|
||||||
|
self.scope.add_statement(stmt)
|
||||||
|
self.freshscope = False
|
||||||
|
else:
|
||||||
|
if token_type not in [tokenize.COMMENT, tokenize.INDENT,
|
||||||
|
tokenize.NEWLINE, tokenize.NL,
|
||||||
|
tokenize.ENDMARKER]:
|
||||||
|
debug.warning('token not classified', tok, token_type,
|
||||||
|
self.start_pos[0])
|
||||||
|
|
||||||
del self.buf
|
del self.buf
|
||||||
return self.module
|
return self.module
|
||||||
|
|||||||
Reference in New Issue
Block a user