Refactored the parser calls. Now it's possible to use jedi.parser.python.parse to quickly parse something.

jedi/parser/__init__.py

@@ -1,2 +1,2 @@
 from jedi.parser.parser import Parser, ParserWithRecovery, ParseError
-from jedi.parser.python import load_grammar
+from jedi.parser.pgen2.pgen import generate_grammar

jedi/parser/parser.py

@@ -68,7 +68,7 @@ class Parser(object):
     }
 
     def __init__(self, grammar, source, start_symbol='file_input',
-                 tokenizer=None, start_parsing=True):
+                 tokens=None, start_parsing=True):
         # Todo Remove start_parsing (with False)
 
         self._used_names = {}
@@ -86,11 +86,11 @@ class Parser(object):
         self._parsed = None
 
         if start_parsing:
-            if tokenizer is None:
-                tokenizer = tokenize.source_tokens(source, use_exact_op_types=True)
-            self.parse(tokenizer)
+            if tokens is None:
+                tokens = tokenize.source_tokens(source, use_exact_op_types=True)
+            self.parse(tokens)
 
-    def parse(self, tokenizer):
+    def parse(self, tokens):
         if self._parsed is not None:
             return self._parsed
 
@@ -100,7 +100,7 @@ class Parser(object):
             self.error_recovery, start_number
         )
 
-        self._parsed = self.pgen_parser.parse(tokenizer)
+        self._parsed = self.pgen_parser.parse(tokens)
 
         if self._start_symbol == 'file_input' != self._parsed.type:
             # If there's only one statement, we get back a non-module. That's
@@ -205,7 +205,7 @@ class ParserWithRecovery(Parser):
     :param module_path: The path of the module in the file system, may be None.
     :type module_path: str
     """
-    def __init__(self, grammar, source, module_path=None, tokenizer=None,
+    def __init__(self, grammar, source, module_path=None, tokens=None,
                  start_parsing=True):
         self.syntax_errors = []
 
@@ -224,7 +224,7 @@ class ParserWithRecovery(Parser):
         # else:
         super(ParserWithRecovery, self).__init__(
             grammar, source,
-            tokenizer=tokenizer,
+            tokens=tokens,
             start_parsing=start_parsing
         )
 

jedi/parser/pgen2/parse.py

@@ -118,8 +118,8 @@ class PgenParser(object):
         self.rootnode = None
         self.error_recovery = error_recovery
 
-    def parse(self, tokenizer):
-        for type_, value, start_pos, prefix in tokenizer:
+    def parse(self, tokens):
+        for type_, value, start_pos, prefix in tokens:
             if self.addtoken(type_, value, start_pos, prefix):
                 break
         else:

jedi/parser/python/__init__.py

@@ -41,13 +41,20 @@ def load_grammar(version=None):
             return load_grammar()
 
 
-def parse(source, grammar=None, error_recovery=False):
+def parse(code, grammar=None, error_recovery=True):
+    added_newline = not code.endswith('\n')
+    if added_newline:
+        code += '\n'
+
     if grammar is None:
         grammar = load_grammar()
 
-    tokens = source_tokens(source)
+    tokens = source_tokens(code, use_exact_op_types=True)
     if error_recovery:
         parser = ParserWithRecovery
     else:
         parser = Parser
-    return parser(grammar, tokens).get_root_node()
+    p = parser(grammar, code, tokens=tokens)
+    if added_newline:
+        p.remove_last_newline()
+    return p.get_root_node()

jedi/parser/tree.py

@@ -14,7 +14,8 @@ The easiest way to play with this module is to use :class:`parsing.Parser`.
 :attr:`parsing.Parser.module` holds an instance of :class:`Module`:
 
 >>> from jedi._compatibility import u
->>> from jedi.parser import ParserWithRecovery, load_grammar
+>>> from jedi.parser.python import load_grammar
+>>> from jedi.parser import ParserWithRecovery
 >>> parser = ParserWithRecovery(load_grammar(), u('import os'), 'example.py')
 >>> submodule = parser.module
 >>> submodule