VimPlug/jedi-fork
1
0
Fork 0
forked from VimPlug/jedi
Code Pull Requests Activity

ignore dedents in tokenizer

Browse Source
This commit is contained in:
Dave Halter
2014-02-20 01:21:20 +01:00
parent 3e9b72b636
commit c26ae3c00d
2 changed files with 9 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
jedi/parser/tokenize.py
View File

@@ -150,7 +150,10 @@ def source_tokens(source, line_offset=0):
def generate_tokens(readline, line_offset=0):
"""The original stdlib Python version with minor modifications"""
"""
The original stdlib Python version with minor modifications.
Modified to not care about dedents.
"""
lnum = line_offset
parenlev = 0
continued = False
@@ -225,7 +228,6 @@ def generate_tokens(readline, line_offset=0):
yield TokenInfo(INDENT, line[:pos], (lnum, 0), (lnum, pos))
while column < indents[-1]:
indents = indents[:-1]
yield TokenInfo(DEDENT, '', (lnum, pos), (lnum, pos))
else: # continued statement
if not line:
@@ -288,8 +290,6 @@ def generate_tokens(readline, line_offset=0):
(lnum, pos), (lnum, pos + 1))
pos += 1
for indent in indents[1:]: # pop remaining indent levels
yield TokenInfo(DEDENT, '', (lnum, 0), (lnum, 0))
yield TokenInfo(ENDMARKER, '', (lnum, 0), (lnum, 0))
@@ -341,8 +341,8 @@ class NoErrorTokenizer(object):
raise common.MultiLevelStopIteration()
# ignore indents/comments
if self.is_fast_parser \
and self.previous[0] in (INDENT, NL, None, NEWLINE, DEDENT) \
and c[0] not in (COMMENT, INDENT, NL, NEWLINE, DEDENT):
and self.previous[0] in (INDENT, NL, None, NEWLINE) \
and c[0] not in (COMMENT, INDENT, NL, NEWLINE):
# print c, tok_name[c[0]]
tok = c[1]