fast parser splitting is now working better

This commit is contained in:
David Halter
2013-05-01 14:31:41 +04:30
parent fa32c0a765
commit f5aaeaa428
2 changed files with 9 additions and 3 deletions

View File

@@ -96,7 +96,7 @@ class NoErrorTokenizer(object):
self.current = self.previous = [None, None, (0, 0), (0, 0), '']
self.in_flow = False
self.new_indent = False
self.parser_indent = 0
self.parser_indent = self.old_parser_indent = 0
self.is_decorator = False
self.first_stmt = True
@@ -153,7 +153,7 @@ class NoErrorTokenizer(object):
if indent < self.parser_indent: # -> dedent
self.parser_indent = indent
self.new_indent = False
if not self.in_flow:
if not self.in_flow or indent < self.old_parser_indent:
close()
self.in_flow = False
elif self.new_indent:
@@ -167,6 +167,7 @@ class NoErrorTokenizer(object):
close()
self.is_decorator = '@' == tok
if not self.is_decorator:
self.old_parser_indent = self.parser_indent
self.parser_indent += 1 # new scope: must be higher
self.new_indent = True

View File

@@ -258,6 +258,7 @@ class FastParser(use_metaclass(CachedFastParser)):
parts = []
is_decorator = False
current_indent = 0
old_indent = 0
new_indent = False
in_flow = False
add_to_last = False
@@ -273,7 +274,7 @@ class FastParser(use_metaclass(CachedFastParser)):
if indent < current_indent: # -> dedent
current_indent = indent
new_indent = False
if not in_flow:
if not in_flow or indent < old_indent:
add_part()
add_to_last = False
in_flow = False
@@ -291,6 +292,7 @@ class FastParser(use_metaclass(CachedFastParser)):
add_to_last = False
is_decorator = '@' == m.group(1)
if not is_decorator:
old_indent = current_indent
current_indent += 1 # it must be higher
new_indent = True
elif is_decorator:
@@ -391,6 +393,9 @@ class FastParser(use_metaclass(CachedFastParser)):
is_fast_parser=True, top_module=self.module)
p.module.parent = self.module
else:
if nodes[index] != self.current_node:
offset = int(nodes[0] == self.current_node)
self.current_node.old_children.pop(index - offset)
node = nodes.pop(index)
p = node.parser
m = p.module