VimPlug/jedi-fork
1
0
Fork 0
forked from VimPlug/jedi
Code Pull Requests Activity

Save a module instead of a parser when pickling.

Browse Source
This commit is contained in:
Dave Halter
2017-03-30 00:55:04 +02:00
parent 932703f04a
commit 8059c3c2c8
9 changed files with 103 additions and 103 deletions
Download Patch File Download Diff File Expand all files Collapse all files

52
jedi/parser/python/__init__.py
View File

@@ -7,7 +7,7 @@ from jedi._compatibility import FileNotFoundError
from jedi.parser.pgen2.pgen import generate_grammar
from jedi.parser.python.parser import Parser, _remove_last_newline
from jedi.parser.python.diff import DiffParser
from jedi.parser.tokenize import source_tokens
from jedi.parser.tokenize import generate_tokens
from jedi.parser import utils
from jedi.common import splitlines, source_to_unicode
@@ -78,42 +78,44 @@ def parse(code=None, path=None, grammar=None, error_recovery=True,
use_cache = cache and path is not None and not code
if use_cache:
# In this case we do actual caching. We just try to load it.
p = utils.load_parser(grammar, path)
if p is not None:
return p.get_root_node()
module_node = utils.load_module(grammar, path)
if module_node is not None:
return module_node
if code is None:
with open(path, 'rb') as f:
code = source_to_unicode(f.read())
added_newline = not code.endswith('\n')
if added_newline:
code += '\n'
tokens = source_tokens(code, use_exact_op_types=True)
# TODO add recovery
p = None
if diff_cache:
try:
parser_cache_item = utils.parser_cache[path]
module_cache_item = utils.parser_cache[path]
except KeyError:
pass
else:
p = parser_cache_item.parser
lines = splitlines(code, keepends=True)
new_node = DiffParser(p).update(lines)
p._parsed = new_node
utils.save_parser(grammar, path, p, pickling=False)
if added_newline:
p.source = code[:-1]
_remove_last_newline(new_node)
module_node = module_cache_item.node
new_node = DiffParser(grammar, module_node).update(
old_lines=module_cache_item.lines,
new_lines=lines
)
utils.save_module(grammar, path, module_node, lines, pickling=False)
return new_node
p = Parser(grammar, code, error_recovery=error_recovery, start_symbol=start_symbol)
new_node = p.parse(tokens=tokens)
added_newline = not code.endswith('\n')
lines = tokenize_lines = splitlines(code, keepends=True)
if added_newline:
_remove_last_newline(new_node)
p.source = code[:-1]
code += '\n'
tokenize_lines = list(tokenize_lines)
tokenize_lines[-1] += '\n'
tokenize_lines.append([])
tokens = generate_tokens(tokenize_lines, use_exact_op_types=True)
p = Parser(grammar, code, error_recovery=error_recovery, start_symbol=start_symbol)
root_node = p.parse(tokens=tokens)
if added_newline:
_remove_last_newline(root_node)
if use_cache or diff_cache:
utils.save_parser(grammar, path, p)
return new_node
utils.save_module(grammar, path, root_node, lines)
return root_node

36
jedi/parser/python/diff.py
View File

@@ -156,10 +156,9 @@ class NewDiffParser(object):
class DiffParser(object):
def __init__(self, parser):
self._parser = parser
self._grammar = self._parser._grammar
self._module = parser.get_root_node()
def __init__(self, grammar, module):
self._grammar = grammar
self._module = module
def _reset(self):
self._copy_count = 0
@@ -167,7 +166,7 @@ class DiffParser(object):
self._nodes_stack = _NodesStack(self._module)
def update(self, lines_new):
def update(self, old_lines, new_lines):
'''
The algorithm works as follows:
@@ -187,24 +186,23 @@ class DiffParser(object):
# Reset the used names cache so they get regenerated.
self._module._used_names = None
self._parser_lines_new = lines_new
self._parser_lines_new = new_lines
self._added_newline = False
if lines_new[-1] != '':
if new_lines[-1] != '':
# The Python grammar needs a newline at the end of a file, but for
# everything else we keep working with lines_new here.
self._parser_lines_new = list(lines_new)
# everything else we keep working with new_lines here.
self._parser_lines_new = list(new_lines)
self._parser_lines_new[-1] += '\n'
self._parser_lines_new.append('')
self._added_newline = True
self._reset()
line_length = len(lines_new)
lines_old = splitlines(self._parser.source, keepends=True)
sm = difflib.SequenceMatcher(None, lines_old, self._parser_lines_new)
line_length = len(new_lines)
sm = difflib.SequenceMatcher(None, old_lines, self._parser_lines_new)
opcodes = sm.get_opcodes()
debug.speed('diff parser calculated')
debug.dbg('diff: line_lengths old: %s, new: %s' % (len(lines_old), line_length))
debug.dbg('diff: line_lengths old: %s, new: %s' % (len(old_lines), line_length))
for operation, i1, i2, j1, j2 in opcodes:
debug.dbg('diff %s old[%s:%s] new[%s:%s]',
@@ -229,17 +227,15 @@ class DiffParser(object):
self._nodes_stack.close()
if self._added_newline:
_remove_last_newline(self._parser.get_root_node())
self._parser.source = ''.join(lines_new)
_remove_last_newline(self._module)
# Good for debugging.
if debug.debug_function:
self._enabled_debugging(lines_old, lines_new)
self._enabled_debugging(old_lines, new_lines)
last_pos = self._module.end_pos[0]
if last_pos != line_length:
current_lines = splitlines(self._module.get_code(), keepends=True)
diff = difflib.unified_diff(current_lines, lines_new)
diff = difflib.unified_diff(current_lines, new_lines)
raise Exception(
"There's an issue (%s != %s) with the diff parser. Please report:\n%s"
% (last_pos, line_length, ''.join(diff))
@@ -248,9 +244,9 @@ class DiffParser(object):
debug.speed('diff parser end')
return self._module
def _enabled_debugging(self, lines_old, lines_new):
def _enabled_debugging(self, old_lines, lines_new):
if self._module.get_code() != ''.join(lines_new):
debug.warning('parser issue:\n%s\n%s', ''.join(lines_old),
debug.warning('parser issue:\n%s\n%s', ''.join(old_lines),
''.join(lines_new))
def _copy_from_old_parser(self, line_offset, until_line_old, until_line_new):

10
jedi/parser/python/parser.py
View File

@@ -53,9 +53,9 @@ class Parser(BaseParser):
self.source = source
self._added_newline = False
# The Python grammar needs a newline at the end of each statement.
if not source.endswith('\n') and start_symbol == 'file_input':
source += '\n'
self._added_newline = True
#if not source.endswith('\n') and start_symbol == 'file_input':
# source += '\n'
# self._added_newline = True
self.new_code = source
@@ -91,8 +91,8 @@ class Parser(BaseParser):
[node]
)
if self._added_newline:
_remove_last_newline(node)
#if self._added_newline:
# _remove_last_newline(node)
return node
def get_root_node(self):