diff --git a/jedi/api/helpers.py b/jedi/api/helpers.py index dbfbdbfa..a17f7808 100644 --- a/jedi/api/helpers.py +++ b/jedi/api/helpers.py @@ -132,7 +132,7 @@ def get_stack_at_position(grammar, code_lines, module_node, pos): safeword = 'ZZZ_USER_WANTS_TO_COMPLETE_HERE_WITH_JEDI' code = code + safeword - p = parser.ParserWithRecovery(grammar, code) + p = parser.Parser(grammar, code, error_recovery=True) try: p.parse(tokens=tokenize_without_endmarker(code)) except EndMarkerReached: diff --git a/jedi/parser/__init__.py b/jedi/parser/__init__.py index 006d930c..731197fa 100644 --- a/jedi/parser/__init__.py +++ b/jedi/parser/__init__.py @@ -1,5 +1,5 @@ from jedi.parser.parser import ParserSyntaxError -from jedi.parser.python.parser import Parser, ParserWithRecovery +from jedi.parser.python.parser import Parser from jedi.parser.pgen2.pgen import generate_grammar from jedi.parser import python diff --git a/jedi/parser/python/__init__.py b/jedi/parser/python/__init__.py index d9b66ad4..f70e4455 100644 --- a/jedi/parser/python/__init__.py +++ b/jedi/parser/python/__init__.py @@ -5,8 +5,7 @@ import os from jedi._compatibility import FileNotFoundError from jedi.parser.pgen2.pgen import generate_grammar -from jedi.parser.python.parser import Parser, ParserWithRecovery, \ - _remove_last_newline +from jedi.parser.python.parser import Parser, _remove_last_newline from jedi.parser.python.diff import DiffParser from jedi.parser.tokenize import source_tokens from jedi.parser import utils @@ -92,13 +91,6 @@ def parse(code=None, path=None, grammar=None, error_recovery=True, code += '\n' tokens = source_tokens(code, use_exact_op_types=True) - kwargs = {} - if error_recovery: - parser = ParserWithRecovery - kwargs = dict() - else: - kwargs = dict(start_symbol=start_symbol) - parser = Parser # TODO add recovery p = None if diff_cache: @@ -116,7 +108,7 @@ def parse(code=None, path=None, grammar=None, error_recovery=True, p.source = code[:-1] _remove_last_newline(new_node) return new_node - p = parser(grammar, code, **kwargs) + p = Parser(grammar, code, error_recovery=error_recovery, start_symbol=start_symbol) new_node = p.parse(tokens=tokens) if added_newline: _remove_last_newline(new_node) diff --git a/jedi/parser/python/diff.py b/jedi/parser/python/diff.py index e0beb794..9ca77315 100644 --- a/jedi/parser/python/diff.py +++ b/jedi/parser/python/diff.py @@ -12,7 +12,7 @@ from collections import namedtuple from jedi._compatibility import use_metaclass from jedi import settings from jedi.common import splitlines -from jedi.parser.python.parser import ParserWithRecovery, _remove_last_newline +from jedi.parser.python.parser import Parser, _remove_last_newline from jedi.parser.python.tree import EndMarker from jedi.parser.utils import parser_cache from jedi import debug @@ -25,7 +25,7 @@ class CachedFastParser(type): def __call__(self, grammar, source, module_path=None): pi = parser_cache.get(module_path, None) if pi is None or not settings.fast_parser: - return ParserWithRecovery(grammar, source, module_path) + return Parser(grammar, source, module_path) parser = pi.parser d = DiffParser(parser) @@ -139,7 +139,7 @@ class NewDiffParser(object): self._path = path from jedi.parser.python import load_grammar grammar = load_grammar(version=python_version) - self._parser = ParserWithRecovery(grammar) + self._parser = Parser(grammar, error_recovery=True) self._module = None def update(self, lines): @@ -374,9 +374,10 @@ class DiffParser(object): until_line, line_offset=parsed_until_line ) - self._active_parser = ParserWithRecovery( + self._active_parser = Parser( self._grammar, source='\n', + error_recovery=True ) return self._active_parser.parse(tokens=tokens) diff --git a/jedi/parser/python/parser.py b/jedi/parser/python/parser.py index 0b99e1d0..5e14ed8a 100644 --- a/jedi/parser/python/parser.py +++ b/jedi/parser/python/parser.py @@ -8,6 +8,14 @@ from jedi.parser.parser import BaseParser class Parser(BaseParser): + """ + This class is used to parse a Python file, it then divides them into a + class structure of different scopes. + + :param grammar: The grammar object of pgen2. Loaded by load_grammar. + :param source: The codebase for the parser. Must be unicode. + """ + node_map = { 'expr_stmt': tree.ExprStmt, 'classdef': tree.Class, @@ -39,8 +47,8 @@ class Parser(BaseParser): } default_node = tree.PythonNode - def __init__(self, grammar, source, start_symbol='file_input'): - super(Parser, self).__init__(grammar, start_symbol) + def __init__(self, grammar, source, error_recovery=True, start_symbol='file_input'): + super(Parser, self).__init__(grammar, start_symbol, error_recovery=error_recovery) self.source = source self._added_newline = False @@ -51,7 +59,27 @@ class Parser(BaseParser): self.new_code = source + self.syntax_errors = [] + self._omit_dedent_list = [] + self._indent_counter = 0 + + # TODO do print absolute import detection here. + # try: + # del python_grammar_no_print_statement.keywords["print"] + # except KeyError: + # pass # Doesn't exist in the Python 3 grammar. + + # if self.options["print_function"]: + # python_grammar = pygram.python_grammar_no_print_statement + # else: + def parse(self, tokens): + if self._error_recovery: + if self._start_symbol != 'file_input': + raise NotImplementedError + + tokens = self._recovery_tokenize(tokens) + node = super(Parser, self).parse(tokens) if self._start_symbol == 'file_input' != node.type: @@ -109,65 +137,6 @@ class Parser(BaseParser): else: return tree.Operator(value, start_pos, prefix) - -def _remove_last_newline(node): - endmarker = node.children[-1] - # The newline is either in the endmarker as a prefix or the previous - # leaf as a newline token. - prefix = endmarker.prefix - if prefix.endswith('\n'): - endmarker.prefix = prefix = prefix[:-1] - last_end = 0 - if '\n' not in prefix: - # Basically if the last line doesn't end with a newline. we - # have to add the previous line's end_position. - previous_leaf = endmarker.get_previous_leaf() - if previous_leaf is not None: - last_end = previous_leaf.end_pos[1] - last_line = re.sub('.*\n', '', prefix) - endmarker.start_pos = endmarker.line - 1, last_end + len(last_line) - else: - newline = endmarker.get_previous_leaf() - if newline is None: - return # This means that the parser is empty. - - assert newline.value.endswith('\n') - newline.value = newline.value[:-1] - endmarker.start_pos = \ - newline.start_pos[0], newline.start_pos[1] + len(newline.value) - - -class ParserWithRecovery(Parser): - """ - This class is used to parse a Python file, it then divides them into a - class structure of different scopes. - - :param grammar: The grammar object of pgen2. Loaded by load_grammar. - :param source: The codebase for the parser. Must be unicode. - """ - def __init__(self, grammar, source): - super(ParserWithRecovery, self).__init__( - grammar, source, - ) - - self.syntax_errors = [] - self._omit_dedent_list = [] - self._indent_counter = 0 - - # TODO do print absolute import detection here. - # try: - # del python_grammar_no_print_statement.keywords["print"] - # except KeyError: - # pass # Doesn't exist in the Python 3 grammar. - - # if self.options["print_function"]: - # python_grammar = pygram.python_grammar_no_print_statement - # else: - - def parse(self, tokens): - root_node = super(ParserWithRecovery, self).parse(self._tokenize(tokens)) - return root_node - def error_recovery(self, grammar, stack, arcs, typ, value, start_pos, prefix, add_token_callback): """ @@ -175,6 +144,11 @@ class ParserWithRecovery(Parser): allows using different grammars (even non-Python). However, error recovery is purely written for Python. """ + if not self._error_recovery: + return super(Parser, self).error_recovery( + grammar, stack, arcs, typ, value, start_pos, prefix, + add_token_callback) + def current_suite(stack): # For now just discard everything that is not a suite or # file_input, if we detect an error. @@ -232,7 +206,7 @@ class ParserWithRecovery(Parser): stack[start_index:] = [] return failed_stack - def _tokenize(self, tokens): + def _recovery_tokenize(self, tokens): for typ, value, start_pos, prefix in tokens: # print(tokenize.tok_name[typ], repr(value), start_pos, repr(prefix)) if typ == DEDENT: @@ -248,3 +222,30 @@ class ParserWithRecovery(Parser): self._indent_counter += 1 yield typ, value, start_pos, prefix + + +def _remove_last_newline(node): + endmarker = node.children[-1] + # The newline is either in the endmarker as a prefix or the previous + # leaf as a newline token. + prefix = endmarker.prefix + if prefix.endswith('\n'): + endmarker.prefix = prefix = prefix[:-1] + last_end = 0 + if '\n' not in prefix: + # Basically if the last line doesn't end with a newline. we + # have to add the previous line's end_position. + previous_leaf = endmarker.get_previous_leaf() + if previous_leaf is not None: + last_end = previous_leaf.end_pos[1] + last_line = re.sub('.*\n', '', prefix) + endmarker.start_pos = endmarker.line - 1, last_end + len(last_line) + else: + newline = endmarker.get_previous_leaf() + if newline is None: + return # This means that the parser is empty. + + assert newline.value.endswith('\n') + newline.value = newline.value[:-1] + endmarker.start_pos = \ + newline.start_pos[0], newline.start_pos[1] + len(newline.value) diff --git a/test/test_parser/test_diff_parser.py b/test/test_parser/test_diff_parser.py index fde7b510..72db9a24 100644 --- a/test/test_parser/test_diff_parser.py +++ b/test/test_parser/test_diff_parser.py @@ -8,7 +8,7 @@ from jedi.common import splitlines from jedi import cache from jedi.parser.python import load_grammar from jedi.parser.python.diff import DiffParser -from jedi.parser import ParserWithRecovery +from jedi.parser import Parser from jedi.parser.tokenize import source_tokens @@ -45,7 +45,7 @@ class Differ(object): def initialize(self, code): debug.dbg('differ: initialize', color='YELLOW') grammar = load_grammar() - self.parser = ParserWithRecovery(grammar, code) + self.parser = Parser(grammar, code, error_recovery=True) tokens = source_tokens(self.parser.new_code, use_exact_op_types=True) return self.parser.parse(tokens)