VimPlug/parso
1
0
Fork 0
mirror of https://github.com/davidhalter/parso.git synced 2026-05-17 05:49:58 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Pass tokens in diff tokenizer

Browse Source
This commit is contained in:
Dave Halter
2020-04-03 01:01:37 +02:00
parent 12063d42fc
commit 62fd03edda
1 changed files with 5 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files
parso/python/diff.py
+5 -4
View File
@@ -409,7 +409,8 @@ class DiffParser(object):
indents=indents indents=indents
) )
stack = self._active_parser.stack stack = self._active_parser.stack
for typ, string, start_pos, prefix in tokens: for token in tokens:
typ = token.type
if typ == PythonTokenTypes.DEDENT: if typ == PythonTokenTypes.DEDENT:
if len(indents) < initial_indentation_count: if len(indents) < initial_indentation_count:
# We are done here, only thing that can come now is an # We are done here, only thing that can come now is an
@@ -431,17 +432,17 @@ class DiffParser(object):
prefix prefix
) )
break break
elif typ == PythonTokenTypes.NEWLINE and start_pos[0] >= until_line: elif typ == PythonTokenTypes.NEWLINE and token.start_pos[0] >= until_line:
was_newline = True was_newline = True
elif was_newline: elif was_newline:
was_newline = False was_newline = False
if len(indents) == initial_indentation_count: if len(indents) == initial_indentation_count:
# Check if the parser is actually in a valid suite state. # Check if the parser is actually in a valid suite state.
if _suite_or_file_input_is_valid(self._pgen_grammar, stack): if _suite_or_file_input_is_valid(self._pgen_grammar, stack):
yield PythonToken(PythonTokenTypes.ENDMARKER, '', start_pos, '') yield PythonToken(PythonTokenTypes.ENDMARKER, '', token.start_pos, '')
break break
yield PythonToken(typ, string, start_pos, prefix) yield token
class _NodesTreeNode(object): class _NodesTreeNode(object):