1
0
forked from VimPlug/jedi

Replace the tokenizer's output with a tuple (switching back from a Token class).

This commit is contained in:
Dave Halter
2014-12-16 00:09:31 +01:00
parent 680fdd574b
commit eaace104dd
5 changed files with 50 additions and 136 deletions

View File

@@ -291,13 +291,10 @@ class Parser(object):
self._scope = self.module
"""
new_scope = False
for token in tokenizer:
typ = token.type
value = token.value
for typ, value, start_pos, prefix in tokenizer:
if typ == tokenize.OP:
typ = grammar.opmap[value]
yield typ, value, token.prefix, token.start_pos
yield typ, value, prefix, start_pos
def __repr__(self):
return "<%s: %s>" % (type(self).__name__, self.module)