VimPlug/parso
1
0
Fork 0
mirror of https://github.com/davidhalter/parso.git synced 2026-02-05 17:38:03 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Define token types a bit different in diff parser

Browse Source
This commit is contained in:
Dave Halter
2020-04-03 01:05:11 +02:00
parent 62fd03edda
commit 8bbd304eb9
1 changed files with 9 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
parso/python/diff.py
View File

@@ -21,6 +21,10 @@ DEBUG_DIFF_PARSER = False
_INDENTATION_TOKENS = 'INDENT', 'ERROR_DEDENT', 'DEDENT' _INDENTATION_TOKENS = 'INDENT', 'ERROR_DEDENT', 'DEDENT'
NEWLINE = PythonTokenTypes.NEWLINE
DEDENT = PythonTokenTypes.DEDENT
ENDMARKER = PythonTokenTypes.ENDMARKER
def _is_indentation_error_leaf(node): def _is_indentation_error_leaf(node):
return node.type == 'error_leaf' and node.token_type in _INDENTATION_TOKENS return node.type == 'error_leaf' and node.token_type in _INDENTATION_TOKENS
@@ -411,13 +415,13 @@ class DiffParser(object):
stack = self._active_parser.stack stack = self._active_parser.stack
for token in tokens: for token in tokens:
typ = token.type typ = token.type
if typ == PythonTokenTypes.DEDENT: if typ == DEDENT:
if len(indents) < initial_indentation_count: if len(indents) < initial_indentation_count:
# We are done here, only thing that can come now is an # We are done here, only thing that can come now is an
# endmarker or another dedented code block. # endmarker or another dedented code block.
while True: while True:
typ, string, start_pos, prefix = next(tokens) typ, string, start_pos, prefix = next(tokens)
if typ != PythonTokenTypes.DEDENT: if typ != DEDENT:
break break
if '\n' in prefix or '\r' in prefix: if '\n' in prefix or '\r' in prefix:
@@ -427,19 +431,19 @@ class DiffParser(object):
if start_pos[1] - len(prefix) == 0: if start_pos[1] - len(prefix) == 0:
prefix = '' prefix = ''
yield PythonToken( yield PythonToken(
PythonTokenTypes.ENDMARKER, '', ENDMARKER, '',
start_pos, start_pos,
prefix prefix
) )
break break
elif typ == PythonTokenTypes.NEWLINE and token.start_pos[0] >= until_line: elif typ == NEWLINE and token.start_pos[0] >= until_line:
was_newline = True was_newline = True
elif was_newline: elif was_newline:
was_newline = False was_newline = False
if len(indents) == initial_indentation_count: if len(indents) == initial_indentation_count:
# Check if the parser is actually in a valid suite state. # Check if the parser is actually in a valid suite state.
if _suite_or_file_input_is_valid(self._pgen_grammar, stack): if _suite_or_file_input_is_valid(self._pgen_grammar, stack):
yield PythonToken(PythonTokenTypes.ENDMARKER, '', token.start_pos, '') yield PythonToken(ENDMARKER, '', token.start_pos, '')
break break
yield token yield token