diff --git a/jedi/evaluate/__init__.py b/jedi/evaluate/__init__.py index 2f2dd334..0a0fa698 100644 --- a/jedi/evaluate/__init__.py +++ b/jedi/evaluate/__init__.py @@ -73,7 +73,6 @@ from itertools import tee, chain from jedi._compatibility import next, hasattr, unicode from jedi.parser import tree as pr -from jedi.parser.pytree import python_symbols from jedi.parser.tokenize import Token from jedi.parser import fast from jedi import debug @@ -178,7 +177,7 @@ class Evaluator(object): elif isinstance(element, pr.Keyword): # For False/True/None return [compiled.builtin.get_by_name(element.value)] - elif element.type == python_symbols.power: + elif element.type == 'power': types = self._eval_atom(element.children[0]) for trailer in element.children[1:]: if trailer == '**': # has a power operation. @@ -186,19 +185,19 @@ class Evaluator(object): types = self.eval_trailer(types, trailer) return types - elif pr.is_node(element, 'testlist_star_expr', 'testlist'): + elif element.type in ('testlist_star_expr', 'testlist',): # The implicit tuple in statements. return [iterable.ImplicitTuple(self, element)] - elif pr.is_node(element, 'not_test') or pr.is_node(element, 'factor'): + elif element.type in ('not_test', 'factor'): types = self.eval_element(element.children[-1]) for operator in element.children[:-1]: types = list(precedence.factor_calculate(self, types, operator)) return types - elif pr.is_node(element, 'test'): + elif element.type == 'test': # `x if foo else y` case. return (self.eval_element(element.children[0]) + self.eval_element(element.children[-1])) - elif pr.is_node(element, 'dotted_name'): + elif element.type == 'dotted_name': types = self._eval_atom(element.children[0]) for next_name in element.children[2::2]: types = list(chain.from_iterable(self.find_types(typ, next_name) diff --git a/jedi/evaluate/representation.py b/jedi/evaluate/representation.py index 70871852..9e884c02 100644 --- a/jedi/evaluate/representation.py +++ b/jedi/evaluate/representation.py @@ -36,7 +36,6 @@ from itertools import chain from jedi._compatibility import use_metaclass, unicode, Python3Method from jedi.parser import tree as pr from jedi.parser.tokenize import Token -from jedi.parser.pytree import python_symbols from jedi import debug from jedi import common from jedi.cache import underscore_memoization @@ -484,7 +483,7 @@ class Function(use_metaclass(CachedMetaClass, Wrapper)): trailer = dec.children[2:-1] if trailer: # Create a trailer and evaluate it. - trailer = pr.Node(python_symbols.trailer, trailer) + trailer = pr.Node('trailer', trailer) dec_results = self._evaluator.eval_trailer(dec_results, trailer) if not len(dec_results): diff --git a/jedi/parser/__init__.py b/jedi/parser/__init__.py index 6e665cde..023c8f41 100644 --- a/jedi/parser/__init__.py +++ b/jedi/parser/__init__.py @@ -22,7 +22,6 @@ from jedi._compatibility import next from jedi import common from jedi.parser import tree as pt from jedi.parser import tokenize -from jedi.parser import pytree from jedi.parser import pgen2 OPERATOR_KEYWORDS = 'and', 'for', 'if', 'else', 'in', 'is', 'lambda', 'not', 'or' @@ -62,7 +61,7 @@ class Parser(object): if not source.endswith('\n'): source += '\n' - _ast_mapping = { + self._ast_mapping = { 'expr_stmt': pt.ExprStmt, 'classdef': pt.Class, 'funcdef': pt.Function, @@ -88,10 +87,15 @@ class Parser(object): 'decorator': pt.Decorator, } - self._ast_mapping = dict((getattr(pytree.python_symbols, k), v) - for k, v in _ast_mapping.items()) - self.global_names = [] + + # TODO do print absolute import detection here. + #try: + # del python_grammar_no_print_statement.keywords["print"] + #except KeyError: + # pass # Doesn't exist in the Python 3 grammar. + + #if self.options["print_function"]: # python_grammar = pygram.python_grammar_no_print_statement #else: @@ -108,6 +112,7 @@ class Parser(object): self.module.used_names = self.used_names self.module.path = module_path self.module.set_global_names(self.global_names) + self.grammar_symbols = grammar.number2symbol def convert_node(self, grammar, type, children): """ @@ -117,20 +122,18 @@ class Parser(object): grammar rule produces a new complete node, so that the tree is build strictly bottom-up. """ - #print(type, children, pytree.type_repr(type)) + symbol = grammar.number2symbol[type] try: - new_node = self._ast_mapping[type](children) + new_node = self._ast_mapping[symbol](children) except KeyError: - new_node = pt.Node(type, children) + new_node = pt.Node(symbol, children) # We need to check raw_node always, because the same node can be # returned by convert multiple times. - if type == pytree.python_symbols.global_stmt: + if symbol == 'global_stmt': self.global_names += new_node.get_defined_names() elif isinstance(new_node, (pt.ClassOrFunc, pt.Module)) \ - and type in (pytree.python_symbols.funcdef, - pytree.python_symbols.classdef, - pytree.python_symbols.file_input): + and symbol in ('funcdef', 'classdef', 'file_input'): # scope_name_stack handling scope_names = self.scope_names_stack.pop() if isinstance(new_node, pt.ClassOrFunc): @@ -180,10 +183,10 @@ class Parser(object): """ # For now just discard everything that is not a suite or # file_input, if we detect an error. - for i, (dfa, state, (type, _)) in reversed(list(enumerate(stack))): + for i, (dfa, state, (_type, _)) in reversed(list(enumerate(stack))): # `suite` can sometimes be only simple_stmt, not stmt. - if type in (grammar.symbol2number['file_input'], - grammar.symbol2number['suite']): + symbol = grammar.number2symbol[_type] + if symbol in ('file_input', 'suite'): index = i break self._stack_removal(stack, index + 1) diff --git a/jedi/parser/pytree.py b/jedi/parser/pytree.py deleted file mode 100644 index 3fe39509..00000000 --- a/jedi/parser/pytree.py +++ /dev/null @@ -1,54 +0,0 @@ -# Copyright 2006 Google, Inc. All Rights Reserved. -# Licensed to PSF under a Contributor Agreement. - -""" -Python parse tree definitions. - -This is a very concrete parse tree; we need to keep every token and -even the comments and whitespace between tokens. - -There's also a pattern matching implementation here. -""" - -__author__ = "Guido van Rossum " - -import os - -from . import pgen2 - -# The grammar file -_GRAMMAR_FILE = os.path.join(os.path.dirname(__file__), "grammar3.4.txt") -python_grammar = pgen2.load_grammar(_GRAMMAR_FILE) - - -class Symbols(object): - def __init__(self, grammar): - """Initializer. - - Creates an attribute for each grammar symbol (nonterminal), - whose value is the symbol's type (an int >= 256). - """ - for name, symbol in grammar.symbol2number.items(): - setattr(self, name, symbol) - - -python_grammar = pgen2.load_grammar(_GRAMMAR_FILE) -python_symbols = Symbols(python_grammar) -python_grammar_no_print_statement = python_grammar.copy() -_type_reprs = {} - -try: - del python_grammar_no_print_statement.keywords["print"] -except KeyError: - pass # Doesn't exist in the Python 3 grammar. - - -def type_repr(type_num): - global _type_reprs - if not _type_reprs: - # printing tokens is possible but not as useful - # from .pgen2 import token // token.__dict__.items(): - for name, val in python_symbols.__dict__.items(): - if type(val) == int: - _type_reprs[val] = name - return _type_reprs.setdefault(type_num, type_num) diff --git a/jedi/parser/tree.py b/jedi/parser/tree.py index 71d4fb25..b86c08f6 100644 --- a/jedi/parser/tree.py +++ b/jedi/parser/tree.py @@ -48,7 +48,6 @@ from jedi._compatibility import (next, Python3Method, encoding, is_py3, literal_eval, use_metaclass) from jedi import debug from jedi import cache -from jedi.parser.pytree import python_symbols, type_repr SCOPE_CONTENTS = 'asserts', 'subscopes', 'imports', 'statements', 'returns' @@ -58,12 +57,9 @@ def is_node(node, *symbol_names): try: type = node.type except AttributeError: - pass + return False else: - for symbol_name in symbol_names: - if getattr(python_symbols, symbol_name) == type: - return True - return False + return type in symbol_names def filter_after_position(names, position): @@ -238,7 +234,7 @@ class Name(Leaf): scope = self.parent while scope.parent is not None: if scope.isinstance(Node): - if scope.type == python_symbols.testlist_comp: + if scope.type == 'testlist_comp': try: if isinstance(scope.children[1], CompFor): return scope.children[1] @@ -441,10 +437,7 @@ class Node(Simple): self.type = type def __repr__(self): - """Return a canonical string representation.""" - return "%s(%s, %r)" % (self.__class__.__name__, - type_repr(self.type), - self.children) + return "%s(%s, %r)" % (self.__class__.__name__, self.type, self.children) class IsScopeMeta(type):