VimPlug/jedi
1
0
Fork 1
mirror of https://github.com/davidhalter/jedi.git synced 2025-12-23 13:51:27 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

small refactorings of the tokenizer

Browse Source
This commit is contained in:
Dave Halter
2014-02-21 17:45:56 +01:00
parent efba0cd80c
commit d9bf9be35e
1 changed files with 48 additions and 50 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
jedi/parser/tokenize.py
View File

@@ -158,9 +158,8 @@ def generate_tokens(readline, line_offset=0):
contstr = ''
contline = None
while True: # loop over lines in stream
try:
line = readline()
except StopIteration:
line = readline() # readline returns empty if it's finished. See StringIO
if not line:
if contstr:
yield TokenInfo(ERRORTOKEN, contstr, strstart, (lnum, pos))
break
@@ -179,12 +178,15 @@ def generate_tokens(readline, line_offset=0):
contstr = contstr + line
contline = contline + line
continue
elif pos == max:
break # Don't really understand why this must be here.
while pos < max:
pseudomatch = pseudoprog.match(line, pos)
if pseudomatch: # scan for tokens
if not pseudomatch: # scan for tokens
yield TokenInfo(ERRORTOKEN, line[pos],
(lnum, pos), (lnum, pos + 1))
pos += 1
continue
start, end = pseudomatch.span(1)
spos, epos, pos = (lnum, start), (lnum, end), end
token, initial = line[start:end], line[start]
@@ -227,10 +229,6 @@ def generate_tokens(readline, line_offset=0):
continue
else:
yield TokenInfo(OP, token, spos, epos)
else:
yield TokenInfo(ERRORTOKEN, line[pos],
(lnum, pos), (lnum, pos + 1))
pos += 1
yield TokenInfo(ENDMARKER, '', (lnum, 0), (lnum, 0))