VimPlug/jedi-fork
1
0
Fork 0
forked from VimPlug/jedi
Code Pull Requests Activity

implement an offset in generate_tokens

Browse Source
This commit is contained in:
Dave Halter
2014-02-17 10:08:32 +01:00
parent bb111daf91
commit 6952596117
1 changed files with 15 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files
jedi/parser/tokenize.py
+15 -12
View File
@@ -142,16 +142,15 @@ del _compile
tabsize = 8 tabsize = 8
class TokenError(Exception): def generate_tokens(readline, offset=(1, 0)):
pass lnum = offset[0] - 1
parenlev = 0
continued = False
def generate_tokens(readline):
lnum = parenlev = continued = 0
numchars = '0123456789' numchars = '0123456789'
contstr, needcont = '', 0 contstr, needcont = '', 0
contline = None contline = None
indents = [0] indents = [0]
first_pass = True
while True: # loop over lines in stream while True: # loop over lines in stream
try: try:
@@ -160,7 +159,11 @@ def generate_tokens(readline):
line = b'' line = b''
lnum += 1 lnum += 1
pos = 0
pos, max = 0, len(line) pos, max = 0, len(line)
if first_pass is True:
pos = offset[1]
first_pass = False
if contstr: # continued string if contstr: # continued string
if not line: if not line:
@@ -204,10 +207,10 @@ def generate_tokens(readline):
if line[pos] == '#': if line[pos] == '#':
comment_token = line[pos:].rstrip('\r\n') comment_token = line[pos:].rstrip('\r\n')
nl_pos = pos + len(comment_token) nl_pos = pos + len(comment_token)
yield TokenInfo(COMMENT, comment_token, yield TokenInfo(COMMENT, comment_token, (lnum, pos),
(lnum, pos), (lnum, pos + len(comment_token))) (lnum, pos + len(comment_token)))
yield TokenInfo(NL, line[nl_pos:], yield TokenInfo(NL, line[nl_pos:], (lnum, nl_pos),
(lnum, nl_pos), (lnum, len(line))) (lnum, len(line)))
else: else:
yield TokenInfo( yield TokenInfo(
(NL, COMMENT)[line[pos] == '#'], line[pos:], (NL, COMMENT)[line[pos] == '#'], line[pos:],
@@ -225,7 +228,7 @@ def generate_tokens(readline):
if not line: if not line:
# basically a statement has not been finished here. # basically a statement has not been finished here.
break break
continued = 0 continued = False
while pos < max: while pos < max:
pseudomatch = pseudoprog.match(line, pos) pseudomatch = pseudoprog.match(line, pos)
@@ -270,7 +273,7 @@ def generate_tokens(readline):
elif initial in namechars: # ordinary name elif initial in namechars: # ordinary name
yield TokenInfo(NAME, token, spos, epos) yield TokenInfo(NAME, token, spos, epos)
elif initial == '\\': # continued stmt elif initial == '\\': # continued stmt
continued = 1 continued = True
else: else:
if initial in '([{': if initial in '([{':
parenlev += 1 parenlev += 1