From 19acdd32b72ec4189d98a9a6fb3b0db1cf8fb471 Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Fri, 17 Oct 2014 01:34:47 +0200 Subject: [PATCH] Fixed issues with the Python3.4 grammar file. The order of symbols matters. 'file_input' needs to be the first symbol. --- jedi/parser/grammar.txt | 12 ++++++------ jedi/parser/grammar3.4.txt | 2 +- jedi/parser/pgen2/grammar.py | 1 + jedi/parser/pgen2/token.py | 3 ++- jedi/parser/pytree.py | 22 +++++++++++++++++----- 5 files changed, 27 insertions(+), 13 deletions(-) diff --git a/jedi/parser/grammar.txt b/jedi/parser/grammar.txt index 1e1f24cf..19a29a55 100644 --- a/jedi/parser/grammar.txt +++ b/jedi/parser/grammar.txt @@ -23,9 +23,9 @@ #diagram:rules # Start symbols for the grammar: -# file_input is a module or sequence of commands read from an input file; -# single_input is a single interactive statement; -# eval_input is the input for the eval() and input() functions. +# file_input is a module or sequence of commands read from an input file; +# single_input is a single interactive statement; +# eval_input is the input for the eval() and input() functions. # NB: compound_stmt in single_input is followed by extra NEWLINE! file_input: (NEWLINE | stmt)* ENDMARKER single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE @@ -88,9 +88,9 @@ while_stmt: 'while' test ':' suite ['else' ':' suite] for_stmt: 'for' exprlist 'in' testlist ':' suite ['else' ':' suite] try_stmt: ('try' ':' suite ((except_clause ':' suite)+ - ['else' ':' suite] - ['finally' ':' suite] | - 'finally' ':' suite)) + ['else' ':' suite] + ['finally' ':' suite] | + 'finally' ':' suite)) with_stmt: 'with' with_item (',' with_item)* ':' suite with_item: test ['as' expr] with_var: 'as' expr diff --git a/jedi/parser/grammar3.4.txt b/jedi/parser/grammar3.4.txt index d7aaffd6..75e9a8f8 100644 --- a/jedi/parser/grammar3.4.txt +++ b/jedi/parser/grammar3.4.txt @@ -15,8 +15,8 @@ # file_input is a module or sequence of commands read from an input file; # eval_input is the input for the eval() functions. # NB: compound_stmt in single_input is followed by extra NEWLINE! -single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE file_input: (NEWLINE | stmt)* ENDMARKER +single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE eval_input: testlist NEWLINE* ENDMARKER decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE diff --git a/jedi/parser/pgen2/grammar.py b/jedi/parser/pgen2/grammar.py index d69934c2..9c20fd48 100644 --- a/jedi/parser/pgen2/grammar.py +++ b/jedi/parser/pgen2/grammar.py @@ -173,6 +173,7 @@ opmap_raw = """ // DOUBLESLASH //= DOUBLESLASHEQUAL -> RARROW +... ELLIPSIS """ opmap = {} diff --git a/jedi/parser/pgen2/token.py b/jedi/parser/pgen2/token.py index 8a00efe3..4187cd10 100755 --- a/jedi/parser/pgen2/token.py +++ b/jedi/parser/pgen2/token.py @@ -62,7 +62,8 @@ COMMENT = 52 NL = 53 RARROW = 54 ERRORTOKEN = 55 -N_TOKENS = 56 +ELLIPSIS = 56 +N_TOKENS = 57 NT_OFFSET = 256 #--end constants-- diff --git a/jedi/parser/pytree.py b/jedi/parser/pytree.py index edc7e05a..75aa6ed2 100644 --- a/jedi/parser/pytree.py +++ b/jedi/parser/pytree.py @@ -12,7 +12,6 @@ There's also a pattern matching implementation here. __author__ = "Guido van Rossum " -import sys import os from . import pgen2 @@ -22,7 +21,7 @@ _type_reprs = {} # The grammar file -_GRAMMAR_FILE = os.path.join(os.path.dirname(__file__), "grammar.txt") +_GRAMMAR_FILE = os.path.join(os.path.dirname(__file__), "grammar3.4.txt") class Symbols(object): @@ -42,7 +41,10 @@ python_grammar = pgen2.load_grammar(_GRAMMAR_FILE) python_symbols = Symbols(python_grammar) python_grammar_no_print_statement = python_grammar.copy() -del python_grammar_no_print_statement.keywords["print"] +try: + del python_grammar_no_print_statement.keywords["print"] +except KeyError: + pass # Doesn't exist in the Python 3 grammar. def type_repr(type_num): @@ -67,12 +69,22 @@ def convert(grammar, raw_node): from jedi.parser import representation as pr _ast_mapping = { - 'simple_stmt': pr.ExprStmt, + 'expr_stmt': pr.ExprStmt, 'classdef': pr.Class, 'funcdef': pr.Function, 'file_input': pr.SubModule, 'import_name': pr.Import, 'import_from': pr.Import, + 'break_stmt': pr.KeywordStatement, + 'continue_stmt': pr.KeywordStatement, + 'return_stmt': pr.KeywordStatement, + 'raise_stmt': pr.KeywordStatement, + 'yield_stmt': pr.KeywordStatement, + 'del_stmt': pr.KeywordStatement, + 'pass_stmt': pr.KeywordStatement, + 'global_stmt': pr.KeywordStatement, + 'nonlocal_stmt': pr.KeywordStatement, + 'assert_stmt': pr.KeywordStatement, } ast_mapping = dict((getattr(python_symbols, k), v) for k, v in _ast_mapping.items()) @@ -83,7 +95,7 @@ def convert(grammar, raw_node): if type in grammar.number2symbol: # If there's exactly one child, return that child instead of # creating a new node. - if len(children) == 1: + if len(children) == 1 and type != 'expr_stmt': return children[0] print(raw_node, type_repr(type)) #import pdb; pdb.set_trace()