From ed928d5d58259e46bbd5175774500f24948ece58 Mon Sep 17 00:00:00 2001 From: David Halter Date: Fri, 2 Nov 2012 18:12:22 +0100 Subject: [PATCH] tokenize not needed anymore in _compatibility --- jedi/_compatibility.py | 5 ----- jedi/parsing.py | 8 +++++--- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/jedi/_compatibility.py b/jedi/_compatibility.py index de031f90..11bd1f89 100644 --- a/jedi/_compatibility.py +++ b/jedi/_compatibility.py @@ -68,11 +68,6 @@ else: eval(compile("""def exec_function(source, global_map): exec source in global_map """, 'blub', 'exec')) -# tokenize function -# TODO remove this, not used anymore -import tokenize -tokenize_func = tokenize.generate_tokens - # StringIO (Python 2.5 has no io module), so use io only for py3k try: from StringIO import StringIO diff --git a/jedi/parsing.py b/jedi/parsing.py index db91ba26..51d7649e 100644 --- a/jedi/parsing.py +++ b/jedi/parsing.py @@ -28,7 +28,7 @@ All those classes are being generated by PyFuzzyParser, which takes python text as input and ignores just all the non-python stuff. Basically you could feed it a perl script, and it should still work (which means throw no error. """ -from _compatibility import (next, literal_eval, tokenize_func, StringIO, +from _compatibility import (next, literal_eval, StringIO, property, is_py3k, cleandoc, Python3Method) import tokenize @@ -1566,7 +1566,8 @@ class PyFuzzyParser(object): debug.warning('indentation error on line %s, ignoring it' % (self.start_pos[0])) self._line_of_tokenize_restart = self.start_pos[0] + 1 - self.gen = PushBackIterator(tokenize_func(self.buf.readline)) + self.gen = PushBackIterator(tokenize.generate_tokens( + self.buf.readline)) return self.next() type, tok, self._tokenize_start_pos, self._tokenize_end_pos, \ @@ -1593,7 +1594,8 @@ class PyFuzzyParser(object): :raises: IndentationError """ self.buf = StringIO(self.code) - self.gen = PushBackIterator(tokenize_func(self.buf.readline)) + self.gen = PushBackIterator(tokenize.generate_tokens( + self.buf.readline)) extended_flow = ['else', 'elif', 'except', 'finally'] statement_toks = ['{', '[', '(', '`']