Hack around the fact that the tokenizers are not really integrated with parsers.

parso/python/fstring.py

@@ -5,6 +5,7 @@ from parso.utils import split_lines
 from parso.python.tokenize import Token
 from parso.python import token
 from parso import parser
+from parso.tree import TypedLeaf
 
 version36 = PythonVersionInfo(3, 6)
 
@@ -19,6 +20,8 @@ class TokenNamespace:
     PYTHON_EXPR = 101
     EXCLAMATION_MARK = 102
 
+    token_map = dict((v, k) for k, v in locals().items())
+
     @classmethod
     def generate_token_id(cls, string):
         if string == '{':
@@ -180,3 +183,8 @@ class Parser(parser.BaseParser):
             node = self.default_node('fstring', [node])
 
         return node
+
+    def convert_leaf(self, pgen_grammar, type, value, prefix, start_pos):
+        # TODO this is so ugly.
+        leaf_type = TokenNamespace.token_map[type]
+        return TypedLeaf(leaf_type, value, start_pos, prefix)

parso/tree.py

@@ -206,6 +206,13 @@ class Leaf(NodeOrLeaf):
         return "<%s: %s>" % (type(self).__name__, self.value)
 
 
+class TypedLeaf(Leaf):
+    __slots__ = ('type',)
+    def __init__(self, type, value, start_pos, prefix=''):
+        super(TypedLeaf, self).__init__(value, start_pos, prefix)
+        self.type = type
+
+
 class BaseNode(NodeOrLeaf):
     """
     The super class for all nodes.
@@ -310,7 +317,7 @@ class ErrorLeaf(Leaf):
     A leaf that is either completely invalid in a language (like `$` in Python)
     or is invalid at that position. Like the star in `1 +* 1`.
     """
-    __slots__ = ('original_type')
+    __slots__ = ('original_type',)
     type = 'error_leaf'
 
     def __init__(self, original_type, value, start_pos, prefix=''):