VimPlug/jedi-fork
1
0
Fork 0
forked from VimPlug/jedi
Code Pull Requests Activity

Check parentheses level in tokenizer.

Browse Source
This commit is contained in:
Dave Halter
2014-11-28 02:14:38 +01:00
parent e1d6511f2f
commit 128dbd34b6
1 changed files with 13 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
jedi/parser/tokenize.py
View File

@@ -224,6 +224,7 @@ def generate_tokens(readline, line_offset=0):
The original stdlib Python version with minor modifications. The original stdlib Python version with minor modifications.
Modified to not care about dedents. Modified to not care about dedents.
""" """
paren_level = 0 # count parentheses
indents = [0] indents = [0]
lnum = line_offset lnum = line_offset
numchars = '0123456789' numchars = '0123456789'
@@ -273,18 +274,19 @@ def generate_tokens(readline, line_offset=0):
if new_line and initial not in '\r\n#': if new_line and initial not in '\r\n#':
new_line = False new_line = False
if start > indents[-1]: if paren_level == 0:
yield Token(INDENT, '', spos, '') if start > indents[-1]:
indents.append(start) yield Token(INDENT, '', spos, '')
while start < indents[-1]: indents.append(start)
yield Token(DEDENT, '', spos, '') while start < indents[-1]:
indents.pop() yield Token(DEDENT, '', spos, '')
indents.pop()
if (initial in numchars or # ordinary number if (initial in numchars or # ordinary number
(initial == '.' and token != '.' and token != '...')): (initial == '.' and token != '.' and token != '...')):
yield Token(NUMBER, token, spos, prefix) yield Token(NUMBER, token, spos, prefix)
elif initial in '\r\n': elif initial in '\r\n':
if not new_line: if not new_line and paren_level == 0:
yield Token(NEWLINE, token, spos, prefix) yield Token(NEWLINE, token, spos, prefix)
new_line = True new_line = True
elif initial == '#': elif initial == '#':
@@ -319,6 +321,10 @@ def generate_tokens(readline, line_offset=0):
elif initial == '\\' and line[start:] == '\\\n': # continued stmt elif initial == '\\' and line[start:] == '\\\n': # continued stmt
continue continue
else: else:
if token in '([{':
paren_level += 1
elif token in ')]}':
paren_level -= 1
yield Token(OP, token, spos, prefix) yield Token(OP, token, spos, prefix)
for indent in indents[1:]: for indent in indents[1:]: