VimPlug/jedi-fork
1
0
Fork 0
forked from VimPlug/jedi
Code Pull Requests Activity

removed line from tokenizer

Browse Source
This commit is contained in:
Dave Halter
2014-02-16 15:28:18 +01:00
parent 22928dbcd0
commit bb111daf91
3 changed files with 25 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
jedi/parser/__init__.py
View File

@@ -401,15 +401,12 @@ class Parser(object):
def __next__(self): def __next__(self):
""" Generate the next tokenize pattern. """ """ Generate the next tokenize pattern. """
try: typ, tok, start_pos, end_pos = next(self._gen)
typ, tok, start_pos, end_pos, self.parserline = next(self._gen) # dedents shouldn't change positions
# dedents shouldn't change positions if typ != tokenize.DEDENT:
if typ != tokenize.DEDENT: self.start_pos = start_pos
self.start_pos = start_pos if typ not in (tokenize.INDENT, tokenize.NEWLINE, tokenize.NL):
if typ not in (tokenize.INDENT, tokenize.NEWLINE, tokenize.NL): self.start_pos, self.end_pos = start_pos, end_pos
self.start_pos, self.end_pos = start_pos, end_pos
except (StopIteration, common.MultiLevelStopIteration):
raise
self._current = typ, tok self._current = typ, tok
return self._current return self._current

37
jedi/parser/tokenize.py
View File

@@ -29,7 +29,7 @@ tok_name[ENCODING] = 'ENCODING'
N_TOKENS += 3 N_TOKENS += 3
class TokenInfo(collections.namedtuple('TokenInfo', 'type string start end line')): class TokenInfo(collections.namedtuple('TokenInfo', 'type string start end')):
def __repr__(self): def __repr__(self):
annotated_type = '%d (%s)' % (self.type, tok_name[self.type]) annotated_type = '%d (%s)' % (self.type, tok_name[self.type])
return ('TokenInfo(type=%s, string=%r, start=%r, end=%r, line=%r)' % return ('TokenInfo(type=%s, string=%r, start=%r, end=%r, line=%r)' %
@@ -169,13 +169,12 @@ def generate_tokens(readline):
endmatch = endprog.match(line) endmatch = endprog.match(line)
if endmatch: if endmatch:
pos = end = endmatch.end(0) pos = end = endmatch.end(0)
yield TokenInfo(STRING, contstr + line[:end], yield TokenInfo(STRING, contstr + line[:end], strstart, (lnum, end))
strstart, (lnum, end), contline + line)
contstr, needcont = '', 0 contstr, needcont = '', 0
contline = None contline = None
elif needcont and line[-2:] != '\\\n' and line[-3:] != '\\\r\n': elif needcont and line[-2:] != '\\\n' and line[-3:] != '\\\r\n':
yield TokenInfo(ERRORTOKEN, contstr + line, yield TokenInfo(ERRORTOKEN, contstr + line,
strstart, (lnum, len(line)), contline) strstart, (lnum, len(line)))
contstr = '' contstr = ''
contline = None contline = None
continue continue
@@ -206,21 +205,21 @@ def generate_tokens(readline):
comment_token = line[pos:].rstrip('\r\n') comment_token = line[pos:].rstrip('\r\n')
nl_pos = pos + len(comment_token) nl_pos = pos + len(comment_token)
yield TokenInfo(COMMENT, comment_token, yield TokenInfo(COMMENT, comment_token,
(lnum, pos), (lnum, pos + len(comment_token)), line) (lnum, pos), (lnum, pos + len(comment_token)))
yield TokenInfo(NL, line[nl_pos:], yield TokenInfo(NL, line[nl_pos:],
(lnum, nl_pos), (lnum, len(line)), line) (lnum, nl_pos), (lnum, len(line)))
else: else:
yield TokenInfo( yield TokenInfo(
(NL, COMMENT)[line[pos] == '#'], line[pos:], (NL, COMMENT)[line[pos] == '#'], line[pos:],
(lnum, pos), (lnum, len(line)), line) (lnum, pos), (lnum, len(line)))
continue continue
if column > indents[-1]: # count indents or dedents if column > indents[-1]: # count indents or dedents
indents.append(column) indents.append(column)
yield TokenInfo(INDENT, line[:pos], (lnum, 0), (lnum, pos), line) yield TokenInfo(INDENT, line[:pos], (lnum, 0), (lnum, pos))
while column < indents[-1]: while column < indents[-1]:
indents = indents[:-1] indents = indents[:-1]
yield TokenInfo(DEDENT, '', (lnum, pos), (lnum, pos), line) yield TokenInfo(DEDENT, '', (lnum, pos), (lnum, pos))
else: # continued statement else: # continued statement
if not line: if not line:
@@ -237,20 +236,20 @@ def generate_tokens(readline):
if (initial in numchars or # ordinary number if (initial in numchars or # ordinary number
(initial == '.' and token != '.' and token != '...')): (initial == '.' and token != '.' and token != '...')):
yield TokenInfo(NUMBER, token, spos, epos, line) yield TokenInfo(NUMBER, token, spos, epos)
elif initial in '\r\n': elif initial in '\r\n':
yield TokenInfo(NL if parenlev > 0 else NEWLINE, yield TokenInfo(NL if parenlev > 0 else NEWLINE,
token, spos, epos, line) token, spos, epos)
elif initial == '#': elif initial == '#':
assert not token.endswith("\n") assert not token.endswith("\n")
yield TokenInfo(COMMENT, token, spos, epos, line) yield TokenInfo(COMMENT, token, spos, epos)
elif token in triple_quoted: elif token in triple_quoted:
endprog = endprogs[token] endprog = endprogs[token]
endmatch = endprog.match(line, pos) endmatch = endprog.match(line, pos)
if endmatch: # all on one line if endmatch: # all on one line
pos = endmatch.end(0) pos = endmatch.end(0)
token = line[start:pos] token = line[start:pos]
yield TokenInfo(STRING, token, spos, (lnum, pos), line) yield TokenInfo(STRING, token, spos, (lnum, pos))
else: else:
strstart = (lnum, start) # multiple lines strstart = (lnum, start) # multiple lines
contstr = line[start:] contstr = line[start:]
@@ -267,9 +266,9 @@ def generate_tokens(readline):
contline = line contline = line
break break
else: # ordinary string else: # ordinary string
yield TokenInfo(STRING, token, spos, epos, line) yield TokenInfo(STRING, token, spos, epos)
elif initial in namechars: # ordinary name elif initial in namechars: # ordinary name
yield TokenInfo(NAME, token, spos, epos, line) yield TokenInfo(NAME, token, spos, epos)
elif initial == '\\': # continued stmt elif initial == '\\': # continued stmt
continued = 1 continued = 1
else: else:
@@ -277,15 +276,15 @@ def generate_tokens(readline):
parenlev += 1 parenlev += 1
elif initial in ')]}': elif initial in ')]}':
parenlev -= 1 parenlev -= 1
yield TokenInfo(OP, token, spos, epos, line) yield TokenInfo(OP, token, spos, epos)
else: else:
yield TokenInfo(ERRORTOKEN, line[pos], yield TokenInfo(ERRORTOKEN, line[pos],
(lnum, pos), (lnum, pos + 1), line) (lnum, pos), (lnum, pos + 1))
pos += 1 pos += 1
for indent in indents[1:]: # pop remaining indent levels for indent in indents[1:]: # pop remaining indent levels
yield TokenInfo(DEDENT, '', (lnum, 0), (lnum, 0), '') yield TokenInfo(DEDENT, '', (lnum, 0), (lnum, 0))
yield TokenInfo(ENDMARKER, '', (lnum, 0), (lnum, 0), '') yield TokenInfo(ENDMARKER, '', (lnum, 0), (lnum, 0))
# From here on we have custom stuff (everything before was originally Python # From here on we have custom stuff (everything before was originally Python

2
jedi/parser/user_context.py
View File

@@ -65,7 +65,7 @@ class UserContext(object):
force_point = False force_point = False
last_type = None last_type = None
try: try:
for token_type, tok, start, end, line in gen: for token_type, tok, start, end in gen:
# print 'tok', token_type, tok, force_point # print 'tok', token_type, tok, force_point
if last_type == token_type == tokenize.NAME: if last_type == token_type == tokenize.NAME:
string += ' ' string += ' '