From 5fd6469b6fa3789532f744cfd2f7ee048487df6a Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Sat, 5 Aug 2017 01:16:30 +0200 Subject: [PATCH] A small improvement to python -m parso.python.tokenize --- parso/python/tokenize.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/parso/python/tokenize.py b/parso/python/tokenize.py index 9d8b6b8..6a73b2d 100644 --- a/parso/python/tokenize.py +++ b/parso/python/tokenize.py @@ -406,9 +406,10 @@ if __name__ == "__main__": else: code = sys.stdin.read() + from parso.utils import source_to_unicode, parse_version_string + if isinstance(code, bytes): - from parso.utils import source_to_unicode code = source_to_unicode(code) - for token in tokenize(code): + for token in tokenize(code, parse_version_string()): print(token)