removed the now redundant token.Token version in favor of tokenize.TokenInfo

2014-02-25 16:38:19 +01:00
parent 2db26abf72
commit 6439d6c848
2 changed files with 5 additions and 118 deletions
@@ -318,9 +318,7 @@ class Parser(object):
                   or tok.string in breaks and level <= 0):
            try:
                # print 'parse_stmt', tok, tokenize.tok_name[token_type]
-                tok_list.append(
-                    token_pr.Token.from_tuple(self._gen.current)
-                )
+                tok_list.append(tok)
                if tok.string == 'as':
                    tok = next(self._gen)
                    if tok.type == tokenize.NAME: