From db1079a7fe9dc5481e8fb52cf220761c74e37c0c Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Mon, 28 Aug 2017 18:32:48 +0200 Subject: [PATCH] Actual support fo backticks. --- parso/python/token.py | 35 ++++++++++++++++------------------- parso/python/tokenize.py | 6 +++--- test/test_pgen2.py | 4 ++++ 3 files changed, 23 insertions(+), 22 deletions(-) diff --git a/parso/python/token.py b/parso/python/token.py index e23f78b..fb590a5 100644 --- a/parso/python/token.py +++ b/parso/python/token.py @@ -1,39 +1,36 @@ from __future__ import absolute_import - -from parso._compatibility import py_version +from itertools import count from token import * +from parso._compatibility import py_version -COMMENT = N_TOKENS + +_counter = count(N_TOKENS) +# Never want to see this thing again. +del N_TOKENS + +COMMENT = next(_counter) tok_name[COMMENT] = 'COMMENT' -N_TOKENS += 1 -NL = N_TOKENS +NL = next(_counter) tok_name[NL] = 'NL' -N_TOKENS += 1 +# Sets the attributes that don't exist in these tok_name versions. if py_version >= 30: - BACKQUOTE = N_TOKENS + BACKQUOTE = next(_counter) tok_name[BACKQUOTE] = 'BACKQUOTE' - N_TOKENS += 1 else: - RARROW = N_TOKENS + RARROW = next(_counter) tok_name[RARROW] = 'RARROW' - N_TOKENS += 1 - ELLIPSIS = N_TOKENS + ELLIPSIS = next(_counter) tok_name[ELLIPSIS] = 'ELLIPSIS' - N_TOKENS += 1 -if not py_version >= 35: - ATEQUAL = N_TOKENS +if py_version < 35: + ATEQUAL = next(_counter) tok_name[ATEQUAL] = 'ATEQUAL' - N_TOKENS += 1 -ERROR_DEDENT = N_TOKENS +ERROR_DEDENT = next(_counter) tok_name[ERROR_DEDENT] = 'ERROR_DEDENT' -N_TOKENS += 1 - - # Map from operator to number (since tokenize doesn't do this) diff --git a/parso/python/tokenize.py b/parso/python/tokenize.py index 8e00cbb..e5411c7 100644 --- a/parso/python/tokenize.py +++ b/parso/python/tokenize.py @@ -18,8 +18,8 @@ from collections import namedtuple import itertools as _itertools from codecs import BOM_UTF8 -from parso.python.token import (tok_name, N_TOKENS, ENDMARKER, STRING, NUMBER, opmap, - NAME, OP, ERRORTOKEN, NEWLINE, INDENT, DEDENT, +from parso.python.token import (tok_name, ENDMARKER, STRING, NUMBER, opmap, + NAME, ERRORTOKEN, NEWLINE, INDENT, DEDENT, ERROR_DEDENT) from parso._compatibility import py_version from parso.utils import split_lines @@ -153,7 +153,7 @@ def _create_token_collection(version_info): # recognized as two instances of =). Operator = group(r"\*\*=?", r">>=?", r"<<=?", r"!=", r"//=?", r"->", - r"[+\-*/%&@|^=<>]=?", + r"[+\-*/%&@`|^=<>]=?", r"~") Bracket = '[][(){}]' diff --git a/test/test_pgen2.py b/test/test_pgen2.py index 37b3d42..d559e5b 100644 --- a/test/test_pgen2.py +++ b/test/test_pgen2.py @@ -250,3 +250,7 @@ def test_multiline_str_literals(each_version): "6f630fad67cda0ee1fb1f562db3aa53e") """ _parse(s, each_version) + + +def test_py2_backticks(works_in_py2): + works_in_py2.parse("`1`")