Merge Parser and ParserWithRecovery.

jedi/api/helpers.py

@@ -132,7 +132,7 @@ def get_stack_at_position(grammar, code_lines, module_node, pos):
     safeword = 'ZZZ_USER_WANTS_TO_COMPLETE_HERE_WITH_JEDI'
     code = code + safeword
 
-    p = parser.ParserWithRecovery(grammar, code)
+    p = parser.Parser(grammar, code, error_recovery=True)
     try:
         p.parse(tokens=tokenize_without_endmarker(code))
     except EndMarkerReached:

jedi/parser/__init__.py

@@ -1,5 +1,5 @@
 from jedi.parser.parser import ParserSyntaxError
-from jedi.parser.python.parser import Parser, ParserWithRecovery
+from jedi.parser.python.parser import Parser
 from jedi.parser.pgen2.pgen import generate_grammar
 from jedi.parser import python
 

jedi/parser/python/__init__.py

@@ -5,8 +5,7 @@ import os
 
 from jedi._compatibility import FileNotFoundError
 from jedi.parser.pgen2.pgen import generate_grammar
-from jedi.parser.python.parser import Parser, ParserWithRecovery, \
+from jedi.parser.python.parser import Parser, _remove_last_newline
-    _remove_last_newline
 from jedi.parser.python.diff import DiffParser
 from jedi.parser.tokenize import source_tokens
 from jedi.parser import utils
@@ -92,13 +91,6 @@ def parse(code=None, path=None, grammar=None, error_recovery=True,
         code += '\n'
 
     tokens = source_tokens(code, use_exact_op_types=True)
-    kwargs = {}
-    if error_recovery:
-        parser = ParserWithRecovery
-        kwargs = dict()
-    else:
-        kwargs = dict(start_symbol=start_symbol)
-        parser = Parser
     # TODO add recovery
     p = None
     if diff_cache:
@@ -116,7 +108,7 @@ def parse(code=None, path=None, grammar=None, error_recovery=True,
                 p.source = code[:-1]
                 _remove_last_newline(new_node)
             return new_node
-    p = parser(grammar, code, **kwargs)
+    p = Parser(grammar, code, error_recovery=error_recovery, start_symbol=start_symbol)
     new_node = p.parse(tokens=tokens)
     if added_newline:
         _remove_last_newline(new_node)

jedi/parser/python/diff.py

@@ -12,7 +12,7 @@ from collections import namedtuple
 from jedi._compatibility import use_metaclass
 from jedi import settings
 from jedi.common import splitlines
-from jedi.parser.python.parser import ParserWithRecovery, _remove_last_newline
+from jedi.parser.python.parser import Parser, _remove_last_newline
 from jedi.parser.python.tree import EndMarker
 from jedi.parser.utils import parser_cache
 from jedi import debug
@@ -25,7 +25,7 @@ class CachedFastParser(type):
     def __call__(self, grammar, source, module_path=None):
         pi = parser_cache.get(module_path, None)
         if pi is None or not settings.fast_parser:
-            return ParserWithRecovery(grammar, source, module_path)
+            return Parser(grammar, source, module_path)
 
         parser = pi.parser
         d = DiffParser(parser)
@@ -139,7 +139,7 @@ class NewDiffParser(object):
         self._path = path
         from jedi.parser.python import load_grammar
         grammar = load_grammar(version=python_version)
-        self._parser = ParserWithRecovery(grammar)
+        self._parser = Parser(grammar, error_recovery=True)
         self._module = None
 
     def update(self, lines):
@@ -374,9 +374,10 @@ class DiffParser(object):
             until_line,
             line_offset=parsed_until_line
         )
-        self._active_parser = ParserWithRecovery(
+        self._active_parser = Parser(
             self._grammar,
             source='\n',
+            error_recovery=True
         )
         return self._active_parser.parse(tokens=tokens)
 

jedi/parser/python/parser.py

@@ -8,6 +8,14 @@ from jedi.parser.parser import BaseParser
 
 
 class Parser(BaseParser):
+    """
+    This class is used to parse a Python file, it then divides them into a
+    class structure of different scopes.
+
+    :param grammar: The grammar object of pgen2. Loaded by load_grammar.
+    :param source: The codebase for the parser. Must be unicode.
+    """
+
     node_map = {
         'expr_stmt': tree.ExprStmt,
         'classdef': tree.Class,
@@ -39,8 +47,8 @@ class Parser(BaseParser):
     }
     default_node = tree.PythonNode
 
-    def __init__(self, grammar, source, start_symbol='file_input'):
+    def __init__(self, grammar, source, error_recovery=True, start_symbol='file_input'):
-        super(Parser, self).__init__(grammar, start_symbol)
+        super(Parser, self).__init__(grammar, start_symbol, error_recovery=error_recovery)
 
         self.source = source
         self._added_newline = False
@@ -51,7 +59,27 @@ class Parser(BaseParser):
 
         self.new_code = source
 
+        self.syntax_errors = []
+        self._omit_dedent_list = []
+        self._indent_counter = 0
+
+        # TODO do print absolute import detection here.
+        # try:
+        #     del python_grammar_no_print_statement.keywords["print"]
+        # except KeyError:
+        #     pass  # Doesn't exist in the Python 3 grammar.
+
+        # if self.options["print_function"]:
+        #     python_grammar = pygram.python_grammar_no_print_statement
+        # else:
+
     def parse(self, tokens):
+        if self._error_recovery:
+            if self._start_symbol != 'file_input':
+                raise NotImplementedError
+
+            tokens = self._recovery_tokenize(tokens)
+
         node = super(Parser, self).parse(tokens)
 
         if self._start_symbol == 'file_input' != node.type:
@@ -109,65 +137,6 @@ class Parser(BaseParser):
         else:
             return tree.Operator(value, start_pos, prefix)
 
-
-def _remove_last_newline(node):
-    endmarker = node.children[-1]
-    # The newline is either in the endmarker as a prefix or the previous
-    # leaf as a newline token.
-    prefix = endmarker.prefix
-    if prefix.endswith('\n'):
-        endmarker.prefix = prefix = prefix[:-1]
-        last_end = 0
-        if '\n' not in prefix:
-            # Basically if the last line doesn't end with a newline. we
-            # have to add the previous line's end_position.
-            previous_leaf = endmarker.get_previous_leaf()
-            if previous_leaf is not None:
-                last_end = previous_leaf.end_pos[1]
-        last_line = re.sub('.*\n', '', prefix)
-        endmarker.start_pos = endmarker.line - 1, last_end + len(last_line)
-    else:
-        newline = endmarker.get_previous_leaf()
-        if newline is None:
-            return  # This means that the parser is empty.
-
-        assert newline.value.endswith('\n')
-        newline.value = newline.value[:-1]
-        endmarker.start_pos = \
-            newline.start_pos[0], newline.start_pos[1] + len(newline.value)
-
-
-class ParserWithRecovery(Parser):
-    """
-    This class is used to parse a Python file, it then divides them into a
-    class structure of different scopes.
-
-    :param grammar: The grammar object of pgen2. Loaded by load_grammar.
-    :param source: The codebase for the parser. Must be unicode.
-    """
-    def __init__(self, grammar, source):
-        super(ParserWithRecovery, self).__init__(
-            grammar, source,
-        )
-
-        self.syntax_errors = []
-        self._omit_dedent_list = []
-        self._indent_counter = 0
-
-        # TODO do print absolute import detection here.
-        # try:
-        #     del python_grammar_no_print_statement.keywords["print"]
-        # except KeyError:
-        #     pass  # Doesn't exist in the Python 3 grammar.
-
-        # if self.options["print_function"]:
-        #     python_grammar = pygram.python_grammar_no_print_statement
-        # else:
-
-    def parse(self, tokens):
-        root_node = super(ParserWithRecovery, self).parse(self._tokenize(tokens))
-        return root_node
-
     def error_recovery(self, grammar, stack, arcs, typ, value, start_pos, prefix,
                        add_token_callback):
         """
@@ -175,6 +144,11 @@ class ParserWithRecovery(Parser):
         allows using different grammars (even non-Python). However, error
         recovery is purely written for Python.
         """
+        if not self._error_recovery:
+            return super(Parser, self).error_recovery(
+                grammar, stack, arcs, typ, value, start_pos, prefix,
+                add_token_callback)
+
         def current_suite(stack):
             # For now just discard everything that is not a suite or
             # file_input, if we detect an error.
@@ -232,7 +206,7 @@ class ParserWithRecovery(Parser):
         stack[start_index:] = []
         return failed_stack
 
-    def _tokenize(self, tokens):
+    def _recovery_tokenize(self, tokens):
         for typ, value, start_pos, prefix in tokens:
             # print(tokenize.tok_name[typ], repr(value), start_pos, repr(prefix))
             if typ == DEDENT:
@@ -248,3 +222,30 @@ class ParserWithRecovery(Parser):
                 self._indent_counter += 1
 
             yield typ, value, start_pos, prefix
+
+
+def _remove_last_newline(node):
+    endmarker = node.children[-1]
+    # The newline is either in the endmarker as a prefix or the previous
+    # leaf as a newline token.
+    prefix = endmarker.prefix
+    if prefix.endswith('\n'):
+        endmarker.prefix = prefix = prefix[:-1]
+        last_end = 0
+        if '\n' not in prefix:
+            # Basically if the last line doesn't end with a newline. we
+            # have to add the previous line's end_position.
+            previous_leaf = endmarker.get_previous_leaf()
+            if previous_leaf is not None:
+                last_end = previous_leaf.end_pos[1]
+        last_line = re.sub('.*\n', '', prefix)
+        endmarker.start_pos = endmarker.line - 1, last_end + len(last_line)
+    else:
+        newline = endmarker.get_previous_leaf()
+        if newline is None:
+            return  # This means that the parser is empty.
+
+        assert newline.value.endswith('\n')
+        newline.value = newline.value[:-1]
+        endmarker.start_pos = \
+            newline.start_pos[0], newline.start_pos[1] + len(newline.value)

test/test_parser/test_diff_parser.py

@@ -8,7 +8,7 @@ from jedi.common import splitlines
 from jedi import cache
 from jedi.parser.python import load_grammar
 from jedi.parser.python.diff import DiffParser
-from jedi.parser import ParserWithRecovery
+from jedi.parser import Parser
 from jedi.parser.tokenize import source_tokens
 
 
@@ -45,7 +45,7 @@ class Differ(object):
     def initialize(self, code):
         debug.dbg('differ: initialize', color='YELLOW')
         grammar = load_grammar()
-        self.parser = ParserWithRecovery(grammar, code)
+        self.parser = Parser(grammar, code, error_recovery=True)
         tokens = source_tokens(self.parser.new_code, use_exact_op_types=True)
         return self.parser.parse(tokens)