VimPlug/jedi
1
0
Fork 1
mirror of https://github.com/davidhalter/jedi.git synced 2025-12-08 06:44:46 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

error tokens only exist in the end in a tokenizer

Browse Source
This commit is contained in:
Dave Halter
2014-02-21 00:09:49 +01:00
parent 3232ae5b0c
commit a69eee5876
2 changed files with 22 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
jedi/parser/tokenize.py
View File

@@ -155,7 +155,7 @@ def generate_tokens(readline, line_offset=0):
lnum = line_offset lnum = line_offset
continued = False continued = False
numchars = '0123456789' numchars = '0123456789'
contstr, needcont = '', 0 contstr, needcont = '', False
contline = None contline = None
indents = [0] indents = [0]
@@ -163,6 +163,10 @@ def generate_tokens(readline, line_offset=0):
try: try:
line = readline() line = readline()
except StopIteration: except StopIteration:
if contstr:
yield TokenInfo(ERRORTOKEN, contstr, strstart, (lnum, pos))
contstr, needcont = '', False
contline = None
line = b'' line = b''
lnum += 1 lnum += 1
@@ -176,14 +180,15 @@ def generate_tokens(readline, line_offset=0):
if endmatch: if endmatch:
pos = end = endmatch.end(0) pos = end = endmatch.end(0)
yield TokenInfo(STRING, contstr + line[:end], strstart, (lnum, end)) yield TokenInfo(STRING, contstr + line[:end], strstart, (lnum, end))
contstr, needcont = '', 0 contstr, needcont = '', False
contline = None contline = None
elif needcont and line[-2:] != '\\\n' and line[-3:] != '\\\r\n': elif needcont and line[-2:] != '\\\n' and line[-3:] != '\\\r\n':
yield TokenInfo(ERRORTOKEN, contstr + line, #yield TokenInfo(ERRORTOKEN, contstr + line,
strstart, (lnum, len(line))) # strstart, (lnum, len(line)))
contstr = '' #contstr = ''
contline = None #contline = None
continue #continue
pass
else: else:
contstr = contstr + line contstr = contstr + line
contline = contline + line contline = contline + line
@@ -264,7 +269,7 @@ def generate_tokens(readline, line_offset=0):
strstart = (lnum, start) strstart = (lnum, start)
endprog = (endprogs[initial] or endprogs[token[1]] or endprog = (endprogs[initial] or endprogs[token[1]] or
endprogs[token[2]]) endprogs[token[2]])
contstr, needcont = line[start:], 1 contstr, needcont = line[start:], True
contline = line contline = line
break break
else: # ordinary string else: # ordinary string

9
test/completion/basic.py
View File

@@ -265,3 +265,12 @@ except ImportError, i_b:
i_b i_b
#? ImportError() #? ImportError()
i_b i_b
# -----------------
# continuations
# -----------------
foo = \
1
#? int()
foo