Fixed issues with the Python3.4 grammar file.

The order of symbols matters. 'file_input' needs to be the first symbol.

jedi/parser/grammar.txt

@@ -23,9 +23,9 @@
 #diagram:rules
 
 # Start symbols for the grammar:
-#	file_input is a module or sequence of commands read from an input file;
-#	single_input is a single interactive statement;
-#	eval_input is the input for the eval() and input() functions.
+#	    file_input is a module or sequence of commands read from an input file;
+#	    single_input is a single interactive statement;
+#	    eval_input is the input for the eval() and input() functions.
 # NB: compound_stmt in single_input is followed by extra NEWLINE!
 file_input: (NEWLINE | stmt)* ENDMARKER
 single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE
@@ -88,9 +88,9 @@ while_stmt: 'while' test ':' suite ['else' ':' suite]
 for_stmt: 'for' exprlist 'in' testlist ':' suite ['else' ':' suite]
 try_stmt: ('try' ':' suite
            ((except_clause ':' suite)+
-	    ['else' ':' suite]
-	    ['finally' ':' suite] |
-	   'finally' ':' suite))
+	        ['else' ':' suite]
+	        ['finally' ':' suite] |
+	       'finally' ':' suite))
 with_stmt: 'with' with_item (',' with_item)*  ':' suite
 with_item: test ['as' expr]
 with_var: 'as' expr

jedi/parser/grammar3.4.txt

@@ -15,8 +15,8 @@
 #       file_input is a module or sequence of commands read from an input file;
 #       eval_input is the input for the eval() functions.
 # NB: compound_stmt in single_input is followed by extra NEWLINE!
-single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE
 file_input: (NEWLINE | stmt)* ENDMARKER
+single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE
 eval_input: testlist NEWLINE* ENDMARKER
 
 decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE

jedi/parser/pgen2/grammar.py

@@ -173,6 +173,7 @@ opmap_raw = """
 // DOUBLESLASH
 //= DOUBLESLASHEQUAL
 -> RARROW
+... ELLIPSIS
 """
 
 opmap = {}

jedi/parser/pgen2/token.py

@@ -62,7 +62,8 @@ COMMENT = 52
 NL = 53
 RARROW = 54
 ERRORTOKEN = 55
-N_TOKENS = 56
+ELLIPSIS = 56
+N_TOKENS = 57
 NT_OFFSET = 256
 #--end constants--
 

jedi/parser/pytree.py

@@ -12,7 +12,6 @@ There's also a pattern matching implementation here.
 
 __author__ = "Guido van Rossum <guido@python.org>"
 
-import sys
 import os
 
 from . import pgen2
@@ -22,7 +21,7 @@ _type_reprs = {}
 
 
 # The grammar file
-_GRAMMAR_FILE = os.path.join(os.path.dirname(__file__), "grammar.txt")
+_GRAMMAR_FILE = os.path.join(os.path.dirname(__file__), "grammar3.4.txt")
 
 
 class Symbols(object):
@@ -42,7 +41,10 @@ python_grammar = pgen2.load_grammar(_GRAMMAR_FILE)
 python_symbols = Symbols(python_grammar)
 
 python_grammar_no_print_statement = python_grammar.copy()
-del python_grammar_no_print_statement.keywords["print"]
+try:
+    del python_grammar_no_print_statement.keywords["print"]
+except KeyError:
+    pass  # Doesn't exist in the Python 3 grammar.
 
 
 def type_repr(type_num):
@@ -67,12 +69,22 @@ def convert(grammar, raw_node):
 
     from jedi.parser import representation as pr
     _ast_mapping = {
-        'simple_stmt': pr.ExprStmt,
+        'expr_stmt': pr.ExprStmt,
         'classdef': pr.Class,
         'funcdef': pr.Function,
         'file_input': pr.SubModule,
         'import_name': pr.Import,
         'import_from': pr.Import,
+        'break_stmt': pr.KeywordStatement,
+        'continue_stmt': pr.KeywordStatement,
+        'return_stmt': pr.KeywordStatement,
+        'raise_stmt': pr.KeywordStatement,
+        'yield_stmt': pr.KeywordStatement,
+        'del_stmt': pr.KeywordStatement,
+        'pass_stmt': pr.KeywordStatement,
+        'global_stmt': pr.KeywordStatement,
+        'nonlocal_stmt': pr.KeywordStatement,
+        'assert_stmt': pr.KeywordStatement,
     }
 
     ast_mapping = dict((getattr(python_symbols, k), v) for k, v in _ast_mapping.items())
@@ -83,7 +95,7 @@ def convert(grammar, raw_node):
     if type in grammar.number2symbol:
         # If there's exactly one child, return that child instead of
         # creating a new node.
-        if len(children) == 1:
+        if len(children) == 1 and type != 'expr_stmt':
             return children[0]
         print(raw_node, type_repr(type))
         #import pdb; pdb.set_trace()