Fix an issue in the fast parser splitting.

This commit is contained in:
Dave Halter
2015-01-30 15:17:38 +01:00
parent 413da3b790
commit 1826f432c8
2 changed files with 11 additions and 3 deletions

View File

@@ -330,6 +330,13 @@ class FastParser(use_metaclass(CachedFastParser)):
self.number_of_splits += 1 self.number_of_splits += 1
return text return text
def just_newlines(current_lines):
for line in current_lines:
line = line.lstrip('\t ')
if line and line[0] not in ('#', '\r'):
return False
return True
# Split only new lines. Distinction between \r\n is the tokenizer's # Split only new lines. Distinction between \r\n is the tokenizer's
# job. # job.
self._lines = source.split('\n') self._lines = source.split('\n')
@@ -365,7 +372,7 @@ class FastParser(use_metaclass(CachedFastParser)):
if m: if m:
in_flow = m.group(1) in FLOWS in_flow = m.group(1) in FLOWS
if not is_decorator and not in_flow: if not is_decorator and not in_flow:
if current_lines: if not just_newlines(current_lines):
yield gen_part() yield gen_part()
is_decorator = '@' == m.group(1) is_decorator = '@' == m.group(1)
if not is_decorator: if not is_decorator:

View File

@@ -181,7 +181,8 @@ def test_nested_funcs():
def test_func_with_for_and_comment(): def test_func_with_for_and_comment():
# The first newline is important, leave it. # The first newline is important, leave it. It should not trigger another
# parser split.
src = dedent("""\ src = dedent("""\
def func(): def func():
@@ -190,7 +191,7 @@ def test_func_with_for_and_comment():
for a in [1]: for a in [1]:
# COMMENT # COMMENT
a""") a""")
check_fp(src, 2, 3) check_fp(src, 2)
check_fp('a\n' + src, 1, 3) check_fp('a\n' + src, 1, 3)