VimPlug/jedi-fork
1
0
Fork 0
forked from VimPlug/jedi
Code Pull Requests Activity

protect start_pos/end_pos in parser to hopefully remove them soon

Browse Source
This commit is contained in:
Dave Halter
2014-02-25 00:27:39 +01:00
parent ef8d3633dd
commit 936c7dfde4
1 changed files with 47 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files

85
jedi/parser/__init__.py
View File

@@ -41,9 +41,9 @@ class Parser(object):
tokenizer=None, top_module=None, offset=0): tokenizer=None, top_module=None, offset=0):
self.no_docstr = no_docstr self.no_docstr = no_docstr
self.start_pos = self.end_pos = 1 + offset, 0 self._start_pos = self._end_pos = 1 + offset, 0
# initialize global Scope # initialize global Scope
self.module = pr.SubModule(module_path, self.start_pos, top_module) self.module = pr.SubModule(module_path, self._start_pos, top_module)
self._scope = self.module self._scope = self.module
tokenizer = tokenizer or tokenize.source_tokens(source) tokenizer = tokenizer or tokenize.source_tokens(source)
@@ -60,7 +60,7 @@ class Parser(object):
pass pass
s = self._scope s = self._scope
while s is not None: while s is not None:
s.end_pos = self.end_pos s.end_pos = self._end_pos
s = s.parent s = s.parent
# clean up unused decorators # clean up unused decorators
@@ -73,10 +73,10 @@ class Parser(object):
# This case is only relevant with the FastTokenizer, because # This case is only relevant with the FastTokenizer, because
# otherwise there's always an EndMarker. # otherwise there's always an EndMarker.
# we added a newline before, so we need to "remove" it again. # we added a newline before, so we need to "remove" it again.
self.end_pos = self._gen.previous[2] self._end_pos = self._gen.previous[2]
self.start_pos = self.module.start_pos self._start_pos = self.module.start_pos
self.module.end_pos = self.end_pos self.module.end_pos = self._end_pos
del self._gen del self._gen
def __repr__(self): def __repr__(self):
@@ -114,17 +114,17 @@ class Parser(object):
# token maybe a name or star # token maybe a name or star
return None, tok return None, tok
append((tok.string, self.start_pos)) append((tok.string, self._start_pos))
first_pos = self.start_pos first_pos = self._start_pos
while True: while True:
end_pos = self.end_pos end_pos = self._end_pos
tok = self.next() tok = self.next()
if tok.string != '.': if tok.string != '.':
break break
tok = self.next() tok = self.next()
if tok.type != tokenize.NAME: if tok.type != tokenize.NAME:
break break
append((tok.string, self.start_pos)) append((tok.string, self._start_pos))
n = pr.Name(self.module, names, first_pos, end_pos) if names else None n = pr.Name(self.module, names, first_pos, end_pos) if names else None
return n, tok return n, tok
@@ -207,13 +207,13 @@ class Parser(object):
:return: Return a Scope representation of the tokens. :return: Return a Scope representation of the tokens.
:rtype: Function :rtype: Function
""" """
first_pos = self.start_pos first_pos = self._start_pos
tok = self.next() tok = self.next()
if tok.type != tokenize.NAME: if tok.type != tokenize.NAME:
return None return None
fname = pr.Name(self.module, [(tok.string, self.start_pos)], self.start_pos, fname = pr.Name(self.module, [(tok.string, self._start_pos)], self._start_pos,
self.end_pos) self._end_pos)
tok = self.next() tok = self.next()
if tok.string != '(': if tok.string != '(':
@@ -245,15 +245,15 @@ class Parser(object):
:return: Return a Scope representation of the tokens. :return: Return a Scope representation of the tokens.
:rtype: Class :rtype: Class
""" """
first_pos = self.start_pos first_pos = self._start_pos
cname = self.next() cname = self.next()
if cname.type != tokenize.NAME: if cname.type != tokenize.NAME:
debug.warning("class: syntax err, token is not a name@%s (%s: %s)", debug.warning("class: syntax err, token is not a name@%s (%s: %s)",
self.start_pos[0], tokenize.tok_name[cname.type], cname.string) self._start_pos[0], tokenize.tok_name[cname.type], cname.string)
return None return None
cname = pr.Name(self.module, [(cname.string, self.start_pos)], cname = pr.Name(self.module, [(cname.string, self._start_pos)],
self.start_pos, self.end_pos) self._start_pos, self._end_pos)
super = [] super = []
_next = self.next() _next = self.next()
@@ -262,7 +262,7 @@ class Parser(object):
_next = self.next() _next = self.next()
if _next.string != ':': if _next.string != ':':
debug.warning("class syntax: %s@%s", cname, self.start_pos[0]) debug.warning("class syntax: %s@%s", cname, self._start_pos[0])
return None return None
return pr.Class(self.module, cname, super, first_pos) return pr.Class(self.module, cname, super, first_pos)
@@ -295,7 +295,7 @@ class Parser(object):
self.next() self.next()
tok = self.next() tok = self.next()
first_pos = self.start_pos first_pos = self._start_pos
opening_brackets = ['{', '(', '['] opening_brackets = ['{', '(', '[']
closing_brackets = ['}', ')', ']'] closing_brackets = ['}', ')', ']']
@@ -374,7 +374,7 @@ class Parser(object):
) )
return None, tok return None, tok
stmt = stmt_class(self.module, tok_list, first_pos, self.end_pos, stmt = stmt_class(self.module, tok_list, first_pos, self._end_pos,
as_names=as_names, as_names=as_names,
names_are_set_vars=names_are_set_vars) names_are_set_vars=names_are_set_vars)
@@ -396,8 +396,8 @@ class Parser(object):
#typ, tok, start_pos, end_pos = next(self._gen) #typ, tok, start_pos, end_pos = next(self._gen)
_current = next(self._gen) _current = next(self._gen)
# dedents shouldn't change positions # dedents shouldn't change positions
self.start_pos = _current.start self._start_pos = _current.start
self.end_pos = _current.end self._end_pos = _current.end
#self._current = typ, tok #self._current = typ, tok
return _current return _current
@@ -430,10 +430,10 @@ class Parser(object):
# check again for unindented stuff. this is true for syntax # check again for unindented stuff. this is true for syntax
# errors. only check for names, because thats relevant here. If # errors. only check for names, because thats relevant here. If
# some docstrings are not indented, I don't care. # some docstrings are not indented, I don't care.
while self.start_pos[1] <= self._scope.start_pos[1] \ while self._start_pos[1] <= self._scope.start_pos[1] \
and (token_type == tokenize.NAME or tok_str in ['(', '['])\ and (token_type == tokenize.NAME or tok_str in ['(', '['])\
and self._scope != self.module: and self._scope != self.module:
self._scope.end_pos = self.start_pos self._scope.end_pos = self._start_pos
self._scope = self._scope.parent self._scope = self._scope.parent
if isinstance(self._scope, pr.Module) \ if isinstance(self._scope, pr.Module) \
and not isinstance(self._scope, pr.SubModule): and not isinstance(self._scope, pr.SubModule):
@@ -443,11 +443,11 @@ class Parser(object):
use_as_parent_scope = self._top_module use_as_parent_scope = self._top_module
else: else:
use_as_parent_scope = self._scope use_as_parent_scope = self._scope
first_pos = self.start_pos first_pos = self._start_pos
if tok_str == 'def': if tok_str == 'def':
func = self._parse_function() func = self._parse_function()
if func is None: if func is None:
debug.warning("function: syntax error@%s", self.start_pos[0]) debug.warning("function: syntax error@%s", self._start_pos[0])
continue continue
self.freshscope = True self.freshscope = True
self._scope = self._scope.add_scope(func, self._decorators) self._scope = self._scope.add_scope(func, self._decorators)
@@ -455,7 +455,7 @@ class Parser(object):
elif tok_str == 'class': elif tok_str == 'class':
cls = self._parse_class() cls = self._parse_class()
if cls is None: if cls is None:
debug.warning("class: syntax error@%s" % self.start_pos[0]) debug.warning("class: syntax error@%s" % self._start_pos[0])
continue continue
self.freshscope = True self.freshscope = True
self._scope = self._scope.add_scope(cls, self._decorators) self._scope = self._scope.add_scope(cls, self._decorators)
@@ -464,14 +464,19 @@ class Parser(object):
elif tok_str == 'import': elif tok_str == 'import':
imports = self._parse_import_list() imports = self._parse_import_list()
for count, (m, alias, defunct) in enumerate(imports): for count, (m, alias, defunct) in enumerate(imports):
e = (alias or m or self).end_pos if alias or m:
end_pos = self.end_pos if count + 1 == len(imports) else e #e = (alias or m or self).end_pos
e = (alias or m).end_pos
else:
# TODO cleanup like e = (alias or name or self._gen.current).end_pos
e = self._gen.current.end
end_pos = self._end_pos if count + 1 == len(imports) else e
i = pr.Import(self.module, first_pos, end_pos, m, i = pr.Import(self.module, first_pos, end_pos, m,
alias, defunct=defunct) alias, defunct=defunct)
self._check_user_stmt(i) self._check_user_stmt(i)
self._scope.add_import(i) self._scope.add_import(i)
if not imports: if not imports:
i = pr.Import(self.module, first_pos, self.end_pos, None, i = pr.Import(self.module, first_pos, self._end_pos, None,
defunct=True) defunct=True)
self._check_user_stmt(i) self._check_user_stmt(i)
self.freshscope = False self.freshscope = False
@@ -492,7 +497,7 @@ class Parser(object):
tok_str = 'import' tok_str = 'import'
mod = None mod = None
if not mod and not relative_count or tok_str != "import": if not mod and not relative_count or tok_str != "import":
debug.warning("from: syntax error@%s", self.start_pos[0]) debug.warning("from: syntax error@%s", self._start_pos[0])
defunct = True defunct = True
if tok_str != 'import': if tok_str != 'import':
self._gen.push_last_back() self._gen.push_last_back()
@@ -501,8 +506,12 @@ class Parser(object):
star = name is not None and name.names[0] == '*' star = name is not None and name.names[0] == '*'
if star: if star:
name = None name = None
e = (alias or name or self).end_pos if alias or name:
end_pos = self.end_pos if count + 1 == len(names) else e e = (alias or name).end_pos
else:
# TODO cleanup like e = (alias or name or self._gen.current).end_pos
e = self._gen.current.end
end_pos = self._end_pos if count + 1 == len(names) else e
i = pr.Import(self.module, first_pos, end_pos, name, i = pr.Import(self.module, first_pos, end_pos, name,
alias, mod, star, relative_count, alias, mod, star, relative_count,
defunct=defunct or defunct2) defunct=defunct or defunct2)
@@ -514,7 +523,7 @@ class Parser(object):
set_stmt, tok = self._parse_statement(added_breaks=['in'], set_stmt, tok = self._parse_statement(added_breaks=['in'],
names_are_set_vars=True) names_are_set_vars=True)
if tok.string != 'in': if tok.string != 'in':
debug.warning('syntax err, for flow incomplete @%s', self.start_pos[0]) debug.warning('syntax err, for flow incomplete @%s', self._start_pos[0])
try: try:
statement, tok = self._parse_statement() statement, tok = self._parse_statement()
@@ -524,7 +533,7 @@ class Parser(object):
f = pr.ForFlow(self.module, s, first_pos, set_stmt) f = pr.ForFlow(self.module, s, first_pos, set_stmt)
self._scope = self._scope.add_statement(f) self._scope = self._scope.add_statement(f)
if tok is None or tok.string != ':': if tok is None or tok.string != ':':
debug.warning('syntax err, for flow started @%s', self.start_pos[0]) debug.warning('syntax err, for flow started @%s', self._start_pos[0])
elif tok_str in ['if', 'while', 'try', 'with'] + extended_flow: elif tok_str in ['if', 'while', 'try', 'with'] + extended_flow:
added_breaks = [] added_breaks = []
command = tok_str command = tok_str
@@ -561,10 +570,10 @@ class Parser(object):
s = self._scope.add_statement(f) s = self._scope.add_statement(f)
self._scope = s self._scope = s
if tok.string != ':': if tok.string != ':':
debug.warning('syntax err, flow started @%s', self.start_pos[0]) debug.warning('syntax err, flow started @%s', self._start_pos[0])
# returns # returns
elif tok_str in ['return', 'yield']: elif tok_str in ['return', 'yield']:
s = self.start_pos s = self._start_pos
self.freshscope = False self.freshscope = False
# add returns to the scope # add returns to the scope
func = self._scope.get_parent_until(pr.Function) func = self._scope.get_parent_until(pr.Function)
@@ -616,7 +625,7 @@ class Parser(object):
else: else:
if token_type not in [tokenize.COMMENT, tokenize.NEWLINE]: if token_type not in [tokenize.COMMENT, tokenize.NEWLINE]:
debug.warning('Token not used: %s %s %s', tok_str, debug.warning('Token not used: %s %s %s', tok_str,
tokenize.tok_name[token_type], self.start_pos) tokenize.tok_name[token_type], self._start_pos)
continue continue
self.no_docstr = False self.no_docstr = False