From 34154d05a0e5c7d40e38bd1455fa046c079a7db1 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Fri, 8 Jun 2018 12:46:16 -0400 Subject: [PATCH] Don't mutate the standard library token.tok_name dictionary (#42) * Don't mutate the standard library token.tok_name dictionary Fixes #41. * More robust test that tok_name isn't mutated This test now works in Python 2.7, and actually tests something in Python 3.7, and it's better anyway because it tests the whole dictionary instead of just one token. * Fix test_tok_name_copied in Python 3.7 and PyPy Apparently Python 3.7 adds N_TOKENS to the tok_name dictionary, and PyPy doesn't have NT_OFFSET in it. --- parso/python/token.py | 2 ++ test/test_tokenize.py | 12 ++++++++++++ 2 files changed, 14 insertions(+) diff --git a/parso/python/token.py b/parso/python/token.py index dd849b0..6f7ad5a 100644 --- a/parso/python/token.py +++ b/parso/python/token.py @@ -4,6 +4,8 @@ from token import * from parso._compatibility import py_version +# Don't mutate the standard library dict +tok_name = tok_name.copy() _counter = count(N_TOKENS) # Never want to see this thing again. diff --git a/test/test_tokenize.py b/test/test_tokenize.py index 6911d99..08590a6 100644 --- a/test/test_tokenize.py +++ b/test/test_tokenize.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 # This file contains Unicode characters. from textwrap import dedent +import tokenize as stdlib_tokenize import pytest @@ -235,3 +236,14 @@ def test_error_string(): assert t1.prefix == ' ' assert t1.string == '"\n' assert endmarker.string == '' + +def test_tok_name_copied(): + # Make sure parso doesn't mutate the standard library + tok_len = len(stdlib_tokenize.tok_name) + correct_len = stdlib_tokenize.N_TOKENS + if 'N_TOKENS' in stdlib_tokenize.tok_name.values(): # Python 3.7 + correct_len += 1 + if 'NT_OFFSET' in stdlib_tokenize.tok_name.values(): # Not there in PyPy + correct_len += 1 + + assert tok_len == correct_len