VimPlug/jedi-fork
1
0
Fork 0
forked from VimPlug/jedi
Code Pull Requests Activity

removed parentheses counting from generate_tokens

Browse Source
This commit is contained in:
Dave Halter
2014-02-20 18:45:22 +01:00
parent 7e651684ff
commit 3232ae5b0c
2 changed files with 6 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files
jedi/parser/__init__.py
+3 -2
View File
@@ -63,7 +63,6 @@ class Parser(object):
while s is not None: while s is not None:
s.end_pos = self.end_pos s.end_pos = self.end_pos
s = s.parent s = s.parent
pass
# clean up unused decorators # clean up unused decorators
for d in self._decorators: for d in self._decorators:
@@ -157,7 +156,9 @@ class Parser(object):
brackets = True brackets = True
token_type, tok = self.next() token_type, tok = self.next()
if brackets and tok == '\n': if brackets and tok == '\n':
self.next() token_type, tok = self.next()
if token_type == tokenize.INDENT:
continue # TODO REMOVE, after removing the indents.
i, token_type, tok = self._parse_dot_name(self._current) i, token_type, tok = self._parse_dot_name(self._current)
if not i: if not i:
defunct = True defunct = True
jedi/parser/tokenize.py
+3 -11
View File
@@ -153,7 +153,6 @@ def generate_tokens(readline, line_offset=0):
Modified to not care about dedents. Modified to not care about dedents.
""" """
lnum = line_offset lnum = line_offset
parenlev = 0
continued = False continued = False
numchars = '0123456789' numchars = '0123456789'
contstr, needcont = '', 0 contstr, needcont = '', 0
@@ -190,7 +189,7 @@ def generate_tokens(readline, line_offset=0):
contline = contline + line contline = contline + line
continue continue
elif parenlev == 0 and not continued: # new statement elif not continued: # new statement
if not line: if not line:
break break
column = 0 column = 0
@@ -216,9 +215,7 @@ def generate_tokens(readline, line_offset=0):
yield TokenInfo(NEWLINE, line[nl_pos:], (lnum, nl_pos), yield TokenInfo(NEWLINE, line[nl_pos:], (lnum, nl_pos),
(lnum, len(line))) (lnum, len(line)))
else: else:
yield TokenInfo( yield TokenInfo(NEWLINE, line[pos:], (lnum, pos), (lnum, len(line)))
(NEWLINE, COMMENT)[line[pos] == '#'], line[pos:],
(lnum, pos), (lnum, len(line)))
continue continue
if column > indents[-1]: # count indents or dedents if column > indents[-1]: # count indents or dedents
@@ -244,8 +241,7 @@ def generate_tokens(readline, line_offset=0):
(initial == '.' and token != '.' and token != '...')): (initial == '.' and token != '.' and token != '...')):
yield TokenInfo(NUMBER, token, spos, epos) yield TokenInfo(NUMBER, token, spos, epos)
elif initial in '\r\n': elif initial in '\r\n':
yield TokenInfo(NEWLINE, yield TokenInfo(NEWLINE, token, spos, epos)
token, spos, epos)
elif initial == '#': elif initial == '#':
assert not token.endswith("\n") assert not token.endswith("\n")
yield TokenInfo(COMMENT, token, spos, epos) yield TokenInfo(COMMENT, token, spos, epos)
@@ -278,10 +274,6 @@ def generate_tokens(readline, line_offset=0):
elif initial == '\\': # continued stmt elif initial == '\\': # continued stmt
continued = True continued = True
else: else:
if initial in '([{':
parenlev += 1
elif initial in ')]}':
parenlev -= 1
yield TokenInfo(OP, token, spos, epos) yield TokenInfo(OP, token, spos, epos)
else: else:
yield TokenInfo(ERRORTOKEN, line[pos], yield TokenInfo(ERRORTOKEN, line[pos],