VimPlug/jedi-fork
1
0
Fork 0
forked from VimPlug/jedi
Code Pull Requests Activity

Merge Parser and ParserWithRecovery.

Browse Source
This commit is contained in:
Dave Halter
2017-03-28 02:08:16 +02:00
parent 1d0796ac07
commit ee47be0140
6 changed files with 74 additions and 80 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
jedi/api/helpers.py
View File

@@ -132,7 +132,7 @@ def get_stack_at_position(grammar, code_lines, module_node, pos):
safeword = 'ZZZ_USER_WANTS_TO_COMPLETE_HERE_WITH_JEDI'
code = code + safeword
p = parser.ParserWithRecovery(grammar, code)
p = parser.Parser(grammar, code, error_recovery=True)
try:
p.parse(tokens=tokenize_without_endmarker(code))
except EndMarkerReached:

2
jedi/parser/__init__.py
View File

@@ -1,5 +1,5 @@
from jedi.parser.parser import ParserSyntaxError
from jedi.parser.python.parser import Parser, ParserWithRecovery
from jedi.parser.python.parser import Parser
from jedi.parser.pgen2.pgen import generate_grammar
from jedi.parser import python

12
jedi/parser/python/__init__.py
View File

@@ -5,8 +5,7 @@ import os
from jedi._compatibility import FileNotFoundError
from jedi.parser.pgen2.pgen import generate_grammar
from jedi.parser.python.parser import Parser, ParserWithRecovery, \
_remove_last_newline
from jedi.parser.python.parser import Parser, _remove_last_newline
from jedi.parser.python.diff import DiffParser
from jedi.parser.tokenize import source_tokens
from jedi.parser import utils
@@ -92,13 +91,6 @@ def parse(code=None, path=None, grammar=None, error_recovery=True,
code += '\n'
tokens = source_tokens(code, use_exact_op_types=True)
kwargs = {}
if error_recovery:
parser = ParserWithRecovery
kwargs = dict()
else:
kwargs = dict(start_symbol=start_symbol)
parser = Parser
# TODO add recovery
p = None
if diff_cache:
@@ -116,7 +108,7 @@ def parse(code=None, path=None, grammar=None, error_recovery=True,
p.source = code[:-1]
_remove_last_newline(new_node)
return new_node
p = parser(grammar, code, **kwargs)
p = Parser(grammar, code, error_recovery=error_recovery, start_symbol=start_symbol)
new_node = p.parse(tokens=tokens)
if added_newline:
_remove_last_newline(new_node)

9
jedi/parser/python/diff.py
View File

@@ -12,7 +12,7 @@ from collections import namedtuple
from jedi._compatibility import use_metaclass
from jedi import settings
from jedi.common import splitlines
from jedi.parser.python.parser import ParserWithRecovery, _remove_last_newline
from jedi.parser.python.parser import Parser, _remove_last_newline
from jedi.parser.python.tree import EndMarker
from jedi.parser.utils import parser_cache
from jedi import debug
@@ -25,7 +25,7 @@ class CachedFastParser(type):
def __call__(self, grammar, source, module_path=None):
pi = parser_cache.get(module_path, None)
if pi is None or not settings.fast_parser:
return ParserWithRecovery(grammar, source, module_path)
return Parser(grammar, source, module_path)
parser = pi.parser
d = DiffParser(parser)
@@ -139,7 +139,7 @@ class NewDiffParser(object):
self._path = path
from jedi.parser.python import load_grammar
grammar = load_grammar(version=python_version)
self._parser = ParserWithRecovery(grammar)
self._parser = Parser(grammar, error_recovery=True)
self._module = None
def update(self, lines):
@@ -374,9 +374,10 @@ class DiffParser(object):
until_line,
line_offset=parsed_until_line
)
self._active_parser = ParserWithRecovery(
self._active_parser = Parser(
self._grammar,
source='\n',
error_recovery=True
)
return self._active_parser.parse(tokens=tokens)

125
jedi/parser/python/parser.py
View File

@@ -8,6 +8,14 @@ from jedi.parser.parser import BaseParser
class Parser(BaseParser):
"""
This class is used to parse a Python file, it then divides them into a
class structure of different scopes.
:param grammar: The grammar object of pgen2. Loaded by load_grammar.
:param source: The codebase for the parser. Must be unicode.
"""
node_map = {
'expr_stmt': tree.ExprStmt,
'classdef': tree.Class,
@@ -39,8 +47,8 @@ class Parser(BaseParser):
}
default_node = tree.PythonNode
def __init__(self, grammar, source, start_symbol='file_input'):
super(Parser, self).__init__(grammar, start_symbol)
def __init__(self, grammar, source, error_recovery=True, start_symbol='file_input'):
super(Parser, self).__init__(grammar, start_symbol, error_recovery=error_recovery)
self.source = source
self._added_newline = False
@@ -51,7 +59,27 @@ class Parser(BaseParser):
self.new_code = source
self.syntax_errors = []
self._omit_dedent_list = []
self._indent_counter = 0
# TODO do print absolute import detection here.
# try:
# del python_grammar_no_print_statement.keywords["print"]
# except KeyError:
# pass # Doesn't exist in the Python 3 grammar.
# if self.options["print_function"]:
# python_grammar = pygram.python_grammar_no_print_statement
# else:
def parse(self, tokens):
if self._error_recovery:
if self._start_symbol != 'file_input':
raise NotImplementedError
tokens = self._recovery_tokenize(tokens)
node = super(Parser, self).parse(tokens)
if self._start_symbol == 'file_input' != node.type:
@@ -109,65 +137,6 @@ class Parser(BaseParser):
else:
return tree.Operator(value, start_pos, prefix)
def _remove_last_newline(node):
endmarker = node.children[-1]
# The newline is either in the endmarker as a prefix or the previous
# leaf as a newline token.
prefix = endmarker.prefix
if prefix.endswith('\n'):
endmarker.prefix = prefix = prefix[:-1]
last_end = 0
if '\n' not in prefix:
# Basically if the last line doesn't end with a newline. we
# have to add the previous line's end_position.
previous_leaf = endmarker.get_previous_leaf()
if previous_leaf is not None:
last_end = previous_leaf.end_pos[1]
last_line = re.sub('.*\n', '', prefix)
endmarker.start_pos = endmarker.line - 1, last_end + len(last_line)
else:
newline = endmarker.get_previous_leaf()
if newline is None:
return # This means that the parser is empty.
assert newline.value.endswith('\n')
newline.value = newline.value[:-1]
endmarker.start_pos = \
newline.start_pos[0], newline.start_pos[1] + len(newline.value)
class ParserWithRecovery(Parser):
"""
This class is used to parse a Python file, it then divides them into a
class structure of different scopes.
:param grammar: The grammar object of pgen2. Loaded by load_grammar.
:param source: The codebase for the parser. Must be unicode.
"""
def __init__(self, grammar, source):
super(ParserWithRecovery, self).__init__(
grammar, source,
)
self.syntax_errors = []
self._omit_dedent_list = []
self._indent_counter = 0
# TODO do print absolute import detection here.
# try:
# del python_grammar_no_print_statement.keywords["print"]
# except KeyError:
# pass # Doesn't exist in the Python 3 grammar.
# if self.options["print_function"]:
# python_grammar = pygram.python_grammar_no_print_statement
# else:
def parse(self, tokens):
root_node = super(ParserWithRecovery, self).parse(self._tokenize(tokens))
return root_node
def error_recovery(self, grammar, stack, arcs, typ, value, start_pos, prefix,
add_token_callback):
"""
@@ -175,6 +144,11 @@ class ParserWithRecovery(Parser):
allows using different grammars (even non-Python). However, error
recovery is purely written for Python.
"""
if not self._error_recovery:
return super(Parser, self).error_recovery(
grammar, stack, arcs, typ, value, start_pos, prefix,
add_token_callback)
def current_suite(stack):
# For now just discard everything that is not a suite or
# file_input, if we detect an error.
@@ -232,7 +206,7 @@ class ParserWithRecovery(Parser):
stack[start_index:] = []
return failed_stack
def _tokenize(self, tokens):
def _recovery_tokenize(self, tokens):
for typ, value, start_pos, prefix in tokens:
# print(tokenize.tok_name[typ], repr(value), start_pos, repr(prefix))
if typ == DEDENT:
@@ -248,3 +222,30 @@ class ParserWithRecovery(Parser):
self._indent_counter += 1
yield typ, value, start_pos, prefix
def _remove_last_newline(node):
endmarker = node.children[-1]
# The newline is either in the endmarker as a prefix or the previous
# leaf as a newline token.
prefix = endmarker.prefix
if prefix.endswith('\n'):
endmarker.prefix = prefix = prefix[:-1]
last_end = 0
if '\n' not in prefix:
# Basically if the last line doesn't end with a newline. we
# have to add the previous line's end_position.
previous_leaf = endmarker.get_previous_leaf()
if previous_leaf is not None:
last_end = previous_leaf.end_pos[1]
last_line = re.sub('.*\n', '', prefix)
endmarker.start_pos = endmarker.line - 1, last_end + len(last_line)
else:
newline = endmarker.get_previous_leaf()
if newline is None:
return # This means that the parser is empty.
assert newline.value.endswith('\n')
newline.value = newline.value[:-1]
endmarker.start_pos = \
newline.start_pos[0], newline.start_pos[1] + len(newline.value)

4
test/test_parser/test_diff_parser.py
View File

@@ -8,7 +8,7 @@ from jedi.common import splitlines
from jedi import cache
from jedi.parser.python import load_grammar
from jedi.parser.python.diff import DiffParser
from jedi.parser import ParserWithRecovery
from jedi.parser import Parser
from jedi.parser.tokenize import source_tokens
@@ -45,7 +45,7 @@ class Differ(object):
def initialize(self, code):
debug.dbg('differ: initialize', color='YELLOW')
grammar = load_grammar()
self.parser = ParserWithRecovery(grammar, code)
self.parser = Parser(grammar, code, error_recovery=True)
tokens = source_tokens(self.parser.new_code, use_exact_op_types=True)
return self.parser.parse(tokens)