diff --git a/_compatibility.py b/_compatibility.py index 14d849e1..d1c79df1 100644 --- a/_compatibility.py +++ b/_compatibility.py @@ -57,10 +57,23 @@ except NameError: def unicode(s): return s.decode("utf-8") -# Borrowed from Ned Batchelder +# exec function if sys.hexversion > 0x03000000: def exec_function(source, global_map): exec(source, global_map) else: eval(compile("""def exec_function(source, global_map): exec source in global_map """, 'blub', 'exec')) + +# tokenize function +import tokenize +if sys.hexversion > 0x03000000: + tokenize_func = tokenize.tokenize +else: + tokenize_func = tokenize.generate_tokens + +# BytesIO (Python 2.5 has no io module) +try: + from cStringIO import StringIO as BytesIO +except ImportError: + from io import BytesIO diff --git a/builtin.py b/builtin.py index 4931400a..94ba8763 100644 --- a/builtin.py +++ b/builtin.py @@ -214,7 +214,7 @@ class Parser(CachedModule): # variables for name, value in stmts.items(): if type(value).__name__ == 'file': - value = 'file' + value = 'open()' elif type(value).__name__ in ['int', 'bool', 'float', 'dict', 'list', 'tuple']: value = repr(value) diff --git a/parsing.py b/parsing.py index 592fa4f3..112525c4 100644 --- a/parsing.py +++ b/parsing.py @@ -31,10 +31,9 @@ Ignored statements: TODO take special care for future imports TODO check meta classes """ -from _compatibility import next, literal_eval +from _compatibility import next, literal_eval, tokenize_func, BytesIO import tokenize -from io import BytesIO import re import debug @@ -1246,13 +1245,7 @@ class PyFuzzyParser(object): :raises: IndentationError """ buf = BytesIO(self.code.encode()) - #print(self.code.encode()) - #print(list(tokenize.tokenize(BytesIO(self.code.encode()).readline))[:9]) - import sys - if sys.hexversion > 0x03000000: - self.gen = tokenize.tokenize(buf.readline) - else: - self.gen = tokenize.generate_tokens(buf.readline) + self.gen = tokenize_func(buf.readline) self.currentscope = self.scope extended_flow = ['else', 'elif', 'except', 'finally'] diff --git a/test/run.py b/test/run.py index dfa720ba..191e3cba 100755 --- a/test/run.py +++ b/test/run.py @@ -2,13 +2,12 @@ import os import sys import re -from io import BytesIO import traceback os.chdir('../') sys.path.append('.') import functions -from _compatibility import unicode +from _compatibility import unicode, BytesIO #functions.set_debug_function(functions.debug.print_to_stdout)