VimPlug/parso
1
0
Fork 0
mirror of https://github.com/davidhalter/parso.git synced 2026-01-19 09:33:06 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Tokenizer: It's now clearer when an error dedent appears

Browse Source
This commit is contained in:
Dave Halter
2020-03-29 13:50:36 +02:00
parent a950b82066
commit 9f8a68677d
4 changed files with 20 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
parso/python/diff.py
View File

@@ -93,9 +93,9 @@ def _assert_nodes_are_equal(node1, node2):
children2 = node2.children
except AttributeError:
assert False, (node1, node2)
assert len(children1) == len(children2)
for n1, n2 in zip(children1, children2):
_assert_nodes_are_equal(n1, n2)
assert len(children1) == len(children2)
def _get_debug_error_message(module, old_lines, new_lines):

13
parso/python/tokenize.py
View File

@@ -398,6 +398,7 @@ def tokenize_lines(lines, version_info, start_pos=(1, 0)):
while start < indents[-1]:
if start > indents[-2]:
yield PythonToken(ERROR_DEDENT, '', (lnum, 0), '')
indents[-1] = start
break
yield PythonToken(DEDENT, '', spos, '')
indents.pop()
@@ -554,14 +555,10 @@ def tokenize_lines(lines, version_info, start_pos=(1, 0)):
fstring_stack[:] = []
paren_level = 0
# We only want to dedent if the token is on a new line.
if re.match(r'[ \f\t]*$', line[:start]):
while True:
indent = indents.pop()
if indent > start:
yield PythonToken(DEDENT, '', spos, '')
else:
indents.append(indent)
break
m = re.match(r'[ \f\t]*$', line[:start])
if m is not None:
for t in dedent_if_necessary(m.end()):
yield t
if is_identifier(token):
yield PythonToken(NAME, token, spos, prefix)
else: