VimPlug/parso
1
0
Fork 0
mirror of https://github.com/davidhalter/parso.git synced 2026-01-11 14:08:42 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Define token types a bit different in diff parser

Browse Source
This commit is contained in:
Dave Halter
2020-04-03 01:05:11 +02:00
parent 62fd03edda
commit 8bbd304eb9
1 changed files with 9 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
parso/python/diff.py
View File

@@ -21,6 +21,10 @@ DEBUG_DIFF_PARSER = False
_INDENTATION_TOKENS = 'INDENT', 'ERROR_DEDENT', 'DEDENT'
NEWLINE = PythonTokenTypes.NEWLINE
DEDENT = PythonTokenTypes.DEDENT
ENDMARKER = PythonTokenTypes.ENDMARKER
def _is_indentation_error_leaf(node):
return node.type == 'error_leaf' and node.token_type in _INDENTATION_TOKENS
@@ -411,13 +415,13 @@ class DiffParser(object):
stack = self._active_parser.stack
for token in tokens:
typ = token.type
if typ == PythonTokenTypes.DEDENT:
if typ == DEDENT:
if len(indents) < initial_indentation_count:
# We are done here, only thing that can come now is an
# endmarker or another dedented code block.
while True:
typ, string, start_pos, prefix = next(tokens)
if typ != PythonTokenTypes.DEDENT:
if typ != DEDENT:
break
if '\n' in prefix or '\r' in prefix:
@@ -427,19 +431,19 @@ class DiffParser(object):
if start_pos[1] - len(prefix) == 0:
prefix = ''
yield PythonToken(
PythonTokenTypes.ENDMARKER, '',
ENDMARKER, '',
start_pos,
prefix
)
break
elif typ == PythonTokenTypes.NEWLINE and token.start_pos[0] >= until_line:
elif typ == NEWLINE and token.start_pos[0] >= until_line:
was_newline = True
elif was_newline:
was_newline = False
if len(indents) == initial_indentation_count:
# Check if the parser is actually in a valid suite state.
if _suite_or_file_input_is_valid(self._pgen_grammar, stack):
yield PythonToken(PythonTokenTypes.ENDMARKER, '', token.start_pos, '')
yield PythonToken(ENDMARKER, '', token.start_pos, '')
break
yield token