From 609ab1ffa9a19b77d1d4d337232197a8cd6fbafe Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Fri, 25 Aug 2017 21:21:57 +0200 Subject: [PATCH] Hack around the fact that the tokenizers are not really integrated with parsers. --- parso/python/fstring.py | 8 ++++++++ parso/tree.py | 9 ++++++++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/parso/python/fstring.py b/parso/python/fstring.py index 5d3513f..58cac7d 100644 --- a/parso/python/fstring.py +++ b/parso/python/fstring.py @@ -5,6 +5,7 @@ from parso.utils import split_lines from parso.python.tokenize import Token from parso.python import token from parso import parser +from parso.tree import TypedLeaf version36 = PythonVersionInfo(3, 6) @@ -19,6 +20,8 @@ class TokenNamespace: PYTHON_EXPR = 101 EXCLAMATION_MARK = 102 + token_map = dict((v, k) for k, v in locals().items()) + @classmethod def generate_token_id(cls, string): if string == '{': @@ -180,3 +183,8 @@ class Parser(parser.BaseParser): node = self.default_node('fstring', [node]) return node + + def convert_leaf(self, pgen_grammar, type, value, prefix, start_pos): + # TODO this is so ugly. + leaf_type = TokenNamespace.token_map[type] + return TypedLeaf(leaf_type, value, start_pos, prefix) diff --git a/parso/tree.py b/parso/tree.py index 2a84784..8730af9 100644 --- a/parso/tree.py +++ b/parso/tree.py @@ -206,6 +206,13 @@ class Leaf(NodeOrLeaf): return "<%s: %s>" % (type(self).__name__, self.value) +class TypedLeaf(Leaf): + __slots__ = ('type',) + def __init__(self, type, value, start_pos, prefix=''): + super(TypedLeaf, self).__init__(value, start_pos, prefix) + self.type = type + + class BaseNode(NodeOrLeaf): """ The super class for all nodes. @@ -310,7 +317,7 @@ class ErrorLeaf(Leaf): A leaf that is either completely invalid in a language (like `$` in Python) or is invalid at that position. Like the star in `1 +* 1`. """ - __slots__ = ('original_type') + __slots__ = ('original_type',) type = 'error_leaf' def __init__(self, original_type, value, start_pos, prefix=''):