VimPlug/parso
1
0
Fork 0
mirror of https://github.com/davidhalter/parso.git synced 2025-12-08 05:34:51 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Fix a diff parser issue.

Browse Source
This commit is contained in:
Dave Halter
2017-07-09 11:55:23 +02:00
parent 5b5e4a0616
commit 9899c703ea
3 changed files with 23 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
parso/python/diff.py
View File

@@ -14,7 +14,7 @@ from parso.utils import splitlines
from parso.python.parser import Parser from parso.python.parser import Parser
from parso.python.tree import EndMarker from parso.python.tree import EndMarker
from parso.python.tokenize import (tokenize_lines, NEWLINE, TokenInfo, from parso.python.tokenize import (tokenize_lines, NEWLINE, TokenInfo,
ENDMARKER, INDENT, DEDENT) ENDMARKER, INDENT, DEDENT, ERRORTOKEN)
def _get_last_line(node_or_leaf): def _get_last_line(node_or_leaf):
@@ -301,7 +301,11 @@ class DiffParser(object):
continue continue
is_first_token = False is_first_token = False
if typ == DEDENT: # In case of omitted_first_indent, it might not be dedented fully.
# However this is a sign for us that a dedent happened.
if typ == DEDENT \
or typ == ERRORTOKEN and not string \
and omitted_first_indent and len(indents) == 1:
indents.pop() indents.pop()
if omitted_first_indent and not indents: if omitted_first_indent and not indents:
# We are done here, only thing that can come now is an # We are done here, only thing that can come now is an

2
parso/python/tokenize.py
View File

@@ -248,7 +248,6 @@ def tokenize_lines(lines):
txt = line[pos:] txt = line[pos:]
if txt.endswith('\n'): if txt.endswith('\n'):
new_line = True new_line = True
# TODO remove prefix?
yield TokenInfo(ERRORTOKEN, txt, (lnum, pos), additional_prefix) yield TokenInfo(ERRORTOKEN, txt, (lnum, pos), additional_prefix)
additional_prefix = '' additional_prefix = ''
break break
@@ -279,6 +278,7 @@ def tokenize_lines(lines):
while start < indents[-1]: while start < indents[-1]:
if start > indents[-2]: if start > indents[-2]:
yield TokenInfo(ERRORTOKEN, '', spos, '') yield TokenInfo(ERRORTOKEN, '', spos, '')
print(spos, repr(line))
break break
yield TokenInfo(DEDENT, '', spos, '') yield TokenInfo(DEDENT, '', spos, '')
indents.pop() indents.pop()

18
test/test_diff_parser.py
View File

@@ -463,6 +463,20 @@ def test_in_parentheses_newlines(differ):
b = 2""") b = 2""")
def test_indentation_issue(differ):
code1 = dedent("""
import module
""")
code2 = dedent("""
class L1:
class L2:
class L3:
def f(): pass
def f(): pass
def f(): pass
def f(): pass
""")
differ.initialize(code1) differ.initialize(code1)
differ.parse(code2, parsers=2, copies=1) differ.parse(code2, parsers=2)
differ.parse(code1, parsers=2, copies=1)