From fd1be02f1e6258ac6a97c85be6063c3f319e4da2 Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Fri, 6 Mar 2015 01:47:37 +0100 Subject: [PATCH] Test for unicode tokens in Python 2.7. --- test/test_parser/test_tokenize.py | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/test/test_parser/test_tokenize.py b/test/test_parser/test_tokenize.py index 32ed1b8d..a1fdd9a8 100644 --- a/test/test_parser/test_tokenize.py +++ b/test/test_parser/test_tokenize.py @@ -1,10 +1,9 @@ # -*- coding: utf-8 # This file contains Unicode characters. from io import StringIO -from token import NEWLINE, STRING, INDENT from jedi._compatibility import u, is_py3 -from jedi.parser.token import NAME +from jedi.parser.token import NAME, OP, NEWLINE, STRING, INDENT from jedi import parser @@ -84,17 +83,15 @@ def 我あφ(): pass ''') fundef_io = StringIO(fundef) + tokens = parser.tokenize.generate_tokens(fundef_io.readline) + token_list = list(tokens) + unicode_token = token_list[1] if is_py3: - tokens = parser.tokenize.generate_tokens(fundef_io.readline) - token_list = list(tokens) - identifier_token = next( - (token for token in token_list if token[1] == '我あφ'), - None - ) - self.assertIsNotNone(identifier_token) - assert identifier_token[0] == NAME + assert unicode_token[0] == NAME else: - pass + # Unicode tokens in Python 2 seem to be identified as operators. + # They will be ignored in the parser, that's ok. + assert unicode_token[0] == OP def test_tokenizer_with_string_literal_backslash():