Fix name tokenizing for Python 2

2025-12-22 04:01:32 +08:00 · 2019-07-13 15:34:23 +02:00
parent 2b8544021f
commit 0a5b5f3346
3 changed files with 17 additions and 6 deletions
--- a/test/test_tokenize.py
+++ b/test/test_tokenize.py
@@ -142,7 +142,7 @@ def test_identifier_contains_unicode():
    else:
        # Unicode tokens in Python 2 seem to be identified as operators.
        # They will be ignored in the parser, that's ok.
-        assert unicode_token[0] == OP
+        assert unicode_token[0] == ERRORTOKEN


 def test_quoted_strings():