Trying to change the symbols in node. They are now strings.

With this change we are finally able to get rid of parser/pytree.py

jedi/evaluate/__init__.py

@@ -73,7 +73,6 @@ from itertools import tee, chain
 
 from jedi._compatibility import next, hasattr, unicode
 from jedi.parser import tree as pr
-from jedi.parser.pytree import python_symbols
 from jedi.parser.tokenize import Token
 from jedi.parser import fast
 from jedi import debug
@@ -178,7 +177,7 @@ class Evaluator(object):
         elif isinstance(element, pr.Keyword):
             # For False/True/None
             return [compiled.builtin.get_by_name(element.value)]
-        elif element.type == python_symbols.power:
+        elif element.type == 'power':
             types = self._eval_atom(element.children[0])
             for trailer in element.children[1:]:
                 if trailer == '**':  # has a power operation.
@@ -186,19 +185,19 @@ class Evaluator(object):
                 types = self.eval_trailer(types, trailer)
 
             return types
-        elif pr.is_node(element, 'testlist_star_expr', 'testlist'):
+        elif element.type in ('testlist_star_expr', 'testlist',):
             # The implicit tuple in statements.
             return [iterable.ImplicitTuple(self, element)]
-        elif pr.is_node(element, 'not_test') or pr.is_node(element, 'factor'):
+        elif element.type in ('not_test', 'factor'):
             types = self.eval_element(element.children[-1])
             for operator in element.children[:-1]:
                 types = list(precedence.factor_calculate(self, types, operator))
             return types
-        elif pr.is_node(element, 'test'):
+        elif element.type == 'test':
             # `x if foo else y` case.
             return (self.eval_element(element.children[0]) +
                     self.eval_element(element.children[-1]))
-        elif pr.is_node(element, 'dotted_name'):
+        elif element.type == 'dotted_name':
             types = self._eval_atom(element.children[0])
             for next_name in element.children[2::2]:
                 types = list(chain.from_iterable(self.find_types(typ, next_name)

jedi/evaluate/representation.py

@@ -36,7 +36,6 @@ from itertools import chain
 from jedi._compatibility import use_metaclass, unicode, Python3Method
 from jedi.parser import tree as pr
 from jedi.parser.tokenize import Token
-from jedi.parser.pytree import python_symbols
 from jedi import debug
 from jedi import common
 from jedi.cache import underscore_memoization
@@ -484,7 +483,7 @@ class Function(use_metaclass(CachedMetaClass, Wrapper)):
                 trailer = dec.children[2:-1]
                 if trailer:
                     # Create a trailer and evaluate it.
-                    trailer = pr.Node(python_symbols.trailer, trailer)
+                    trailer = pr.Node('trailer', trailer)
                     dec_results = self._evaluator.eval_trailer(dec_results, trailer)
 
                 if not len(dec_results):

jedi/parser/__init__.py

@@ -22,7 +22,6 @@ from jedi._compatibility import next
 from jedi import common
 from jedi.parser import tree as pt
 from jedi.parser import tokenize
-from jedi.parser import pytree
 from jedi.parser import pgen2
 
 OPERATOR_KEYWORDS = 'and', 'for', 'if', 'else', 'in', 'is', 'lambda', 'not', 'or'
@@ -62,7 +61,7 @@ class Parser(object):
         if not source.endswith('\n'):
             source += '\n'
 
-        _ast_mapping = {
+        self._ast_mapping = {
             'expr_stmt': pt.ExprStmt,
             'classdef': pt.Class,
             'funcdef': pt.Function,
@@ -88,10 +87,15 @@ class Parser(object):
             'decorator': pt.Decorator,
         }
 
-        self._ast_mapping = dict((getattr(pytree.python_symbols, k), v)
-                                 for k, v in _ast_mapping.items())
-
         self.global_names = []
+
+        # TODO do print absolute import detection here.
+        #try:
+        #    del python_grammar_no_print_statement.keywords["print"]
+        #except KeyError:
+        #    pass  # Doesn't exist in the Python 3 grammar.
+
+
         #if self.options["print_function"]:
         #    python_grammar = pygram.python_grammar_no_print_statement
         #else:
@@ -108,6 +112,7 @@ class Parser(object):
         self.module.used_names = self.used_names
         self.module.path = module_path
         self.module.set_global_names(self.global_names)
+        self.grammar_symbols = grammar.number2symbol
 
     def convert_node(self, grammar, type, children):
         """
@@ -117,20 +122,18 @@ class Parser(object):
         grammar rule produces a new complete node, so that the tree is build
         strictly bottom-up.
         """
-        #print(type, children, pytree.type_repr(type))
+        symbol = grammar.number2symbol[type]
         try:
-            new_node = self._ast_mapping[type](children)
+            new_node = self._ast_mapping[symbol](children)
         except KeyError:
-            new_node = pt.Node(type, children)
+            new_node = pt.Node(symbol, children)
 
         # We need to check raw_node always, because the same node can be
         # returned by convert multiple times.
-        if type == pytree.python_symbols.global_stmt:
+        if symbol == 'global_stmt':
             self.global_names += new_node.get_defined_names()
         elif isinstance(new_node, (pt.ClassOrFunc, pt.Module)) \
-                and type in (pytree.python_symbols.funcdef,
+                and symbol in ('funcdef', 'classdef', 'file_input'):
-                             pytree.python_symbols.classdef,
-                             pytree.python_symbols.file_input):
             # scope_name_stack handling
             scope_names = self.scope_names_stack.pop()
             if isinstance(new_node, pt.ClassOrFunc):
@@ -180,10 +183,10 @@ class Parser(object):
         """
         # For now just discard everything that is not a suite or
         # file_input, if we detect an error.
-        for i, (dfa, state, (type, _)) in reversed(list(enumerate(stack))):
+        for i, (dfa, state, (_type, _)) in reversed(list(enumerate(stack))):
             # `suite` can sometimes be only simple_stmt, not stmt.
-            if type in (grammar.symbol2number['file_input'],
+            symbol = grammar.number2symbol[_type]
-                        grammar.symbol2number['suite']):
+            if symbol in ('file_input', 'suite'):
                 index = i
                 break
         self._stack_removal(stack, index + 1)

jedi/parser/pytree.py

@@ -1,54 +0,0 @@
-# Copyright 2006 Google, Inc. All Rights Reserved.
-# Licensed to PSF under a Contributor Agreement.
-
-"""
-Python parse tree definitions.
-
-This is a very concrete parse tree; we need to keep every token and
-even the comments and whitespace between tokens.
-
-There's also a pattern matching implementation here.
-"""
-
-__author__ = "Guido van Rossum <guido@python.org>"
-
-import os
-
-from . import pgen2
-
-# The grammar file
-_GRAMMAR_FILE = os.path.join(os.path.dirname(__file__), "grammar3.4.txt")
-python_grammar = pgen2.load_grammar(_GRAMMAR_FILE)
-
-
-class Symbols(object):
-    def __init__(self, grammar):
-        """Initializer.
-
-        Creates an attribute for each grammar symbol (nonterminal),
-        whose value is the symbol's type (an int >= 256).
-        """
-        for name, symbol in grammar.symbol2number.items():
-            setattr(self, name, symbol)
-
-
-python_grammar = pgen2.load_grammar(_GRAMMAR_FILE)
-python_symbols = Symbols(python_grammar)
-python_grammar_no_print_statement = python_grammar.copy()
-_type_reprs = {}
-
-try:
-    del python_grammar_no_print_statement.keywords["print"]
-except KeyError:
-    pass  # Doesn't exist in the Python 3 grammar.
-
-
-def type_repr(type_num):
-    global _type_reprs
-    if not _type_reprs:
-        # printing tokens is possible but not as useful
-        # from .pgen2 import token // token.__dict__.items():
-        for name, val in python_symbols.__dict__.items():
-            if type(val) == int:
-                _type_reprs[val] = name
-    return _type_reprs.setdefault(type_num, type_num)

jedi/parser/tree.py

@@ -48,7 +48,6 @@ from jedi._compatibility import (next, Python3Method, encoding, is_py3,
                                  literal_eval, use_metaclass)
 from jedi import debug
 from jedi import cache
-from jedi.parser.pytree import python_symbols, type_repr
 
 
 SCOPE_CONTENTS = 'asserts', 'subscopes', 'imports', 'statements', 'returns'
@@ -58,12 +57,9 @@ def is_node(node, *symbol_names):
     try:
         type = node.type
     except AttributeError:
-        pass
+        return False
     else:
-        for symbol_name in symbol_names:
+        return type in symbol_names
-            if getattr(python_symbols, symbol_name) == type:
-                return True
-    return False
 
 
 def filter_after_position(names, position):
@@ -238,7 +234,7 @@ class Name(Leaf):
         scope = self.parent
         while scope.parent is not None:
             if scope.isinstance(Node):
-                if scope.type == python_symbols.testlist_comp:
+                if scope.type == 'testlist_comp':
                     try:
                         if isinstance(scope.children[1], CompFor):
                             return scope.children[1]
@@ -441,10 +437,7 @@ class Node(Simple):
         self.type = type
 
     def __repr__(self):
-        """Return a canonical string representation."""
+        return "%s(%s, %r)" % (self.__class__.__name__, self.type, self.children)
-        return "%s(%s, %r)" % (self.__class__.__name__,
-                               type_repr(self.type),
-                               self.children)
 
 
 class IsScopeMeta(type):