VimPlug/jedi
1
0
Fork 1
mirror of https://github.com/davidhalter/jedi.git synced 2025-12-23 05:41:27 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

small refactorings of the tokenizer

Browse Source
This commit is contained in:
Dave Halter
2014-02-21 17:45:56 +01:00
parent efba0cd80c
commit d9bf9be35e
1 changed files with 48 additions and 50 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
jedi/parser/tokenize.py
View File

@@ -158,9 +158,8 @@ def generate_tokens(readline, line_offset=0):
contstr = '' contstr = ''
contline = None contline = None
while True: # loop over lines in stream while True: # loop over lines in stream
try: line = readline() # readline returns empty if it's finished. See StringIO
line = readline() if not line:
except StopIteration:
if contstr: if contstr:
yield TokenInfo(ERRORTOKEN, contstr, strstart, (lnum, pos)) yield TokenInfo(ERRORTOKEN, contstr, strstart, (lnum, pos))
break break
@@ -179,12 +178,15 @@ def generate_tokens(readline, line_offset=0):
contstr = contstr + line contstr = contstr + line
contline = contline + line contline = contline + line
continue continue
elif pos == max:
break # Don't really understand why this must be here.
while pos < max: while pos < max:
pseudomatch = pseudoprog.match(line, pos) pseudomatch = pseudoprog.match(line, pos)
if pseudomatch: # scan for tokens if not pseudomatch: # scan for tokens
yield TokenInfo(ERRORTOKEN, line[pos],
(lnum, pos), (lnum, pos + 1))
pos += 1
continue
start, end = pseudomatch.span(1) start, end = pseudomatch.span(1)
spos, epos, pos = (lnum, start), (lnum, end), end spos, epos, pos = (lnum, start), (lnum, end), end
token, initial = line[start:end], line[start] token, initial = line[start:end], line[start]
@@ -227,10 +229,6 @@ def generate_tokens(readline, line_offset=0):
continue continue
else: else:
yield TokenInfo(OP, token, spos, epos) yield TokenInfo(OP, token, spos, epos)
else:
yield TokenInfo(ERRORTOKEN, line[pos],
(lnum, pos), (lnum, pos + 1))
pos += 1
yield TokenInfo(ENDMARKER, '', (lnum, 0), (lnum, 0)) yield TokenInfo(ENDMARKER, '', (lnum, 0), (lnum, 0))