diff --git a/parso/python/diff.py b/parso/python/diff.py index 2fbf1f7..a98ae79 100644 --- a/parso/python/diff.py +++ b/parso/python/diff.py @@ -14,7 +14,7 @@ from parso.utils import splitlines from parso.python.parser import Parser from parso.python.tree import EndMarker from parso.python.tokenize import (tokenize_lines, NEWLINE, TokenInfo, - ENDMARKER, INDENT, DEDENT) + ENDMARKER, INDENT, DEDENT, ERRORTOKEN) def _get_last_line(node_or_leaf): @@ -301,7 +301,11 @@ class DiffParser(object): continue is_first_token = False - if typ == DEDENT: + # In case of omitted_first_indent, it might not be dedented fully. + # However this is a sign for us that a dedent happened. + if typ == DEDENT \ + or typ == ERRORTOKEN and not string \ + and omitted_first_indent and len(indents) == 1: indents.pop() if omitted_first_indent and not indents: # We are done here, only thing that can come now is an diff --git a/parso/python/tokenize.py b/parso/python/tokenize.py index 2dffa73..a8a6ac2 100644 --- a/parso/python/tokenize.py +++ b/parso/python/tokenize.py @@ -248,7 +248,6 @@ def tokenize_lines(lines): txt = line[pos:] if txt.endswith('\n'): new_line = True - # TODO remove prefix? yield TokenInfo(ERRORTOKEN, txt, (lnum, pos), additional_prefix) additional_prefix = '' break @@ -279,6 +278,7 @@ def tokenize_lines(lines): while start < indents[-1]: if start > indents[-2]: yield TokenInfo(ERRORTOKEN, '', spos, '') + print(spos, repr(line)) break yield TokenInfo(DEDENT, '', spos, '') indents.pop() diff --git a/test/test_diff_parser.py b/test/test_diff_parser.py index 09c96af..ceca69b 100644 --- a/test/test_diff_parser.py +++ b/test/test_diff_parser.py @@ -463,6 +463,20 @@ def test_in_parentheses_newlines(differ): b = 2""") +def test_indentation_issue(differ): + code1 = dedent(""" + import module + """) + + code2 = dedent(""" + class L1: + class L2: + class L3: + def f(): pass + def f(): pass + def f(): pass + def f(): pass + """) + differ.initialize(code1) - differ.parse(code2, parsers=2, copies=1) - differ.parse(code1, parsers=2, copies=1) + differ.parse(code2, parsers=2)