Introduce TokenTypes

parso/pgen2/grammar.py

@@ -16,9 +16,6 @@ fallback token code OP, but the parser needs the actual token code.
 
 """
 
-from parso.python import token
-
-
 class DFAPlan(object):
     def __init__(self, next_dfa, dfa_pushes=[]):
         self.next_dfa = next_dfa
@@ -111,7 +108,6 @@ class Grammar(object):
 
             # A named token (e.g. NAME, NUMBER, STRING)
             itoken = getattr(self._token_namespace, label, None)
-            assert isinstance(itoken, int), label
             if itoken in self.tokens:
                 return self.tokens[itoken]
             else:
@@ -126,7 +122,7 @@ class Grammar(object):
             if value in self.reserved_syntax_strings:
                 return self.reserved_syntax_strings[value]
             else:
-                self.labels.append((token.NAME, value))
+                self.labels.append(('XXX', value))
                 self.reserved_syntax_strings[value] = ilabel
                 return self.reserved_syntax_strings[value]
 

parso/pgen2/grammar_parser.py

@@ -5,9 +5,9 @@
 # Copyright David Halter and Contributors
 # Modifications are dual-licensed: MIT and PSF.
 
-from parso.python import tokenize
+from parso.python.tokenize import tokenize
 from parso.utils import parse_version_string
-from parso.python import token
+from parso.python.token import PythonTokenTypes
 
 
 class GrammarParser():
@@ -16,7 +16,7 @@ class GrammarParser():
     """
     def __init__(self, bnf_grammar):
         self._bnf_grammar = bnf_grammar
-        self.generator = tokenize.tokenize(
+        self.generator = tokenize(
             bnf_grammar,
             version_info=parse_version_string('3.6')
         )
@@ -24,16 +24,16 @@ class GrammarParser():
 
     def parse(self):
         # grammar: (NEWLINE | rule)* ENDMARKER
-        while self.type != token.ENDMARKER:
-            while self.type == token.NEWLINE:
+        while self.type != PythonTokenTypes.ENDMARKER:
+            while self.type == PythonTokenTypes.NEWLINE:
                 self._gettoken()
 
             # rule: NAME ':' rhs NEWLINE
-            self._current_rule_name = self._expect(token.NAME)
-            self._expect(token.OP, ':')
+            self._current_rule_name = self._expect(PythonTokenTypes.NAME)
+            self._expect(PythonTokenTypes.OP, ':')
 
             a, z = self._parse_rhs()
-            self._expect(token.NEWLINE)
+            self._expect(PythonTokenTypes.NEWLINE)
 
             yield a, z
 
@@ -60,7 +60,8 @@ class GrammarParser():
     def _parse_items(self):
         # items: item+
         a, b = self._parse_item()
-        while self.type in (token.NAME, token.STRING) or self.value in ('(', '['):
+        while self.type in (PythonTokenTypes.NAME, PythonTokenTypes.STRING) \
+                or self.value in ('(', '['):
             c, d = self._parse_item()
             # Need to end on the next item.
             b.add_arc(c)
@@ -72,7 +73,7 @@ class GrammarParser():
         if self.value == "[":
             self._gettoken()
             a, z = self._parse_rhs()
-            self._expect(token.OP, ']')
+            self._expect(PythonTokenTypes.OP, ']')
             # Make it also possible that there is no token and change the
             # state.
             a.add_arc(z)
@@ -97,9 +98,9 @@ class GrammarParser():
         if self.value == "(":
             self._gettoken()
             a, z = self._parse_rhs()
-            self._expect(token.OP, ')')
+            self._expect(PythonTokenTypes.OP, ')')
             return a, z
-        elif self.type in (token.NAME, token.STRING):
+        elif self.type in (PythonTokenTypes.NAME, PythonTokenTypes.STRING):
             a = NFAState(self._current_rule_name)
             z = NFAState(self._current_rule_name)
             # Make it clear that the state transition requires that value.
@@ -110,10 +111,10 @@ class GrammarParser():
             self._raise_error("expected (...) or NAME or STRING, got %s/%s",
                               self.type, self.value)
 
-    def _expect(self, type, value=None):
-        if self.type != type:
-            self._raise_error("expected %s(%s), got %s(%s)",
-                              type, token.tok_name[type], self.type, self.value)
+    def _expect(self, type_, value=None):
+        if self.type != type_:
+            self._raise_error("expected %s, got %s [%s]",
+                              type_, self.type, self.value)
         if value is not None and self.value != value:
             self._raise_error("expected %s, got %s", value, self.value)
         value = self.value

parso/pgen2/parse.py

@@ -14,8 +14,6 @@ See Parser/parser.c in the Python distribution for additional info on
 how this parsing engine works.
 """
 
-from parso.python import tokenize
-
 
 class InternalParseError(Exception):
     """
@@ -24,9 +22,9 @@ class InternalParseError(Exception):
     wrong.
     """
 
-    def __init__(self, msg, type, value, start_pos):
+    def __init__(self, msg, type_, value, start_pos):
         Exception.__init__(self, "%s: type=%r, value=%r, start_pos=%r" %
-                           (msg, tokenize.tok_name[type], value, start_pos))
+                           (msg, type_.name, value, start_pos))
         self.msg = msg
         self.type = type
         self.value = value
@@ -69,9 +67,7 @@ class StackNode(object):
 
 def token_to_ilabel(grammar, type_, value):
     # Map from token to label
-    # TODO this is not good, shouldn't use tokenize.NAME, but somehow use the
-    # grammar.
-    if type_ in (tokenize.NAME, tokenize.OP):
+    if type_.contains_syntax:
         # Check for reserved words (keywords)
         try:
             return grammar.reserved_syntax_strings[value]
@@ -196,6 +192,7 @@ class PgenParser(object):
         # creating a new node.  We still create expr_stmt and
         # file_input though, because a lot of Jedi depends on its
         # logic.
+        print(tos.nodes)
         if len(tos.nodes) == 1:
             new_node = tos.nodes[0]
         else: