VimPlug/parso
1
0
Fork 0
mirror of https://github.com/davidhalter/parso.git synced 2025-12-06 12:54:29 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Starting to refactor the normalizer to make sure that comments are also checked for indentation.

Browse Source 
Using a linked list now for the indentation stack.
This commit is contained in:
Dave Halter
2017-07-02 18:58:43 +02:00
parent 7a7ad2038d
commit 36a59a3d3b
4 changed files with 120 additions and 122 deletions
Download Patch File Download Diff File Expand all files Collapse all files

211
parso/python/normalizer.py
View File

@@ -106,8 +106,9 @@ class IndentationTypes(object):
class IndentationNode(object): class IndentationNode(object):
type = IndentationTypes.SUITE type = IndentationTypes.SUITE
def __init__(self, config, indentation_level): def __init__(self, config, indentation, parent=None):
self.bracket_indentation = self.indentation = config.indentation * indentation_level self.bracket_indentation = self.indentation = indentation
self.parent = parent
def __repr__(self): def __repr__(self):
return '<%s>' % self.__class__.__name__ return '<%s>' % self.__class__.__name__
@@ -121,7 +122,7 @@ class IndentationStack(list):
class BracketNode(IndentationNode): class BracketNode(IndentationNode):
def __init__(self, config, parent_indentation, leaf): def __init__(self, config, parent_indentation, leaf, parent):
self.leaf = leaf self.leaf = leaf
next_leaf = leaf.get_next_leaf() next_leaf = leaf.get_next_leaf()
@@ -148,6 +149,7 @@ class BracketNode(IndentationNode):
self.indentation = ' ' * expected_end_indent self.indentation = ' ' * expected_end_indent
self.bracket_indentation = self.indentation self.bracket_indentation = self.indentation
self.type = IndentationTypes.VERTICAL_BRACKET self.type = IndentationTypes.VERTICAL_BRACKET
self.parent = parent
class ImplicitNode(BracketNode): class ImplicitNode(BracketNode):
@@ -155,8 +157,8 @@ class ImplicitNode(BracketNode):
Implicit indentation after keyword arguments, default arguments, Implicit indentation after keyword arguments, default arguments,
annotations and dict values. annotations and dict values.
""" """
def __init__(self, config, parent_indentation, leaf): def __init__(self, config, parent_indentation, leaf, parent):
super(ImplicitNode, self).__init__(config, parent_indentation, leaf) super(ImplicitNode, self).__init__(config, parent_indentation, leaf, parent)
self.type = IndentationTypes.IMPLICIT self.type = IndentationTypes.IMPLICIT
next_leaf = leaf.get_next_leaf() next_leaf = leaf.get_next_leaf()
@@ -167,7 +169,7 @@ class ImplicitNode(BracketNode):
class BackslashNode(IndentationNode): class BackslashNode(IndentationNode):
type = IndentationTypes.BACKSLASH type = IndentationTypes.BACKSLASH
def __init__(self, config, parent_indentation, containing_leaf): def __init__(self, config, parent_indentation, containing_leaf, parent=None):
from parso.python.tree import search_ancestor from parso.python.tree import search_ancestor
expr_stmt = search_ancestor(containing_leaf, 'expr_stmt') expr_stmt = search_ancestor(containing_leaf, 'expr_stmt')
if expr_stmt is not None: if expr_stmt is not None:
@@ -181,6 +183,7 @@ class BackslashNode(IndentationNode):
self.indentation = ' ' * (equals.end_pos[1] + 1) self.indentation = ' ' * (equals.end_pos[1] + 1)
else: else:
self.bracket_indentation = self.indentation = parent_indentation + config.indentation self.bracket_indentation = self.indentation = parent_indentation + config.indentation
self.parent = parent
def _is_magic_name(name): def _is_magic_name(name):
@@ -191,12 +194,12 @@ class PEP8Normalizer(Normalizer):
def __init__(self, config): def __init__(self, config):
super(PEP8Normalizer, self).__init__(config) super(PEP8Normalizer, self).__init__(config)
self._previous_leaf = None self._previous_leaf = None
self._last_indentation_level = 0
self._on_newline = True self._on_newline = True
self._new_statement = True
self._implicit_indentation_possible = False self._implicit_indentation_possible = False
self._indentation_stack = IndentationStack( # The top of stack of the indentation nodes.
[IndentationNode(config, indentation_level=0)] self._indentation_tos = self._last_indentation_tos = \
) IndentationNode(config, indentation='')
self._in_suite_introducer = False self._in_suite_introducer = False
if ' ' in config.indentation: if ' ' in config.indentation:
@@ -278,22 +281,24 @@ class PEP8Normalizer(Normalizer):
if in_introducer: if in_introducer:
self._in_suite_introducer = True self._in_suite_introducer = True
elif typ == 'suite': elif typ == 'suite':
if self._indentation_stack[-1].type == IndentationTypes.BACKSLASH: if self._indentation_tos.type == IndentationTypes.BACKSLASH:
self._indentation_stack.pop() self._indentation_tos = self._indentation_tos.parent
self._indentation_stack.append( self._indentation_tos = IndentationNode(
IndentationNode(self._config, len(self._indentation_stack)) self._config,
self._indentation_tos.indentation + self._config.indentation,
parent=self._indentation_tos
) )
elif implicit_indentation_possible: elif implicit_indentation_possible:
self._implicit_indentation_possible = True self._implicit_indentation_possible = True
yield yield
if typ == 'suite': if typ == 'suite':
assert self._indentation_stack[-1].type == IndentationTypes.SUITE assert self._indentation_tos.type == IndentationTypes.SUITE
self._indentation_stack.pop() self._indentation_tos = self._indentation_tos.parent
elif implicit_indentation_possible: elif implicit_indentation_possible:
self._implicit_indentation_possible = False self._implicit_indentation_possible = False
if self._indentation_stack[-1].type == IndentationTypes.IMPLICIT: if self._indentation_tos.type == IndentationTypes.IMPLICIT:
self._indentation_stack.pop() self._indentation_tos = self._indentation_tos.parent
elif in_introducer: elif in_introducer:
self._in_suite_introducer = False self._in_suite_introducer = False
@@ -309,15 +314,20 @@ class PEP8Normalizer(Normalizer):
# This part is used for the part call after for. # This part is used for the part call after for.
break break
self._old_normalize(part, part.create_spacing_part()) self._old_normalize(part, part.create_spacing_part())
return self._old_normalize(leaf, part)
x = self._old_normalize(leaf, part)
self._last_indentation_tos = self._indentation_tos
self._new_statement = leaf.type == 'newline'
return x
def _old_normalize(self, leaf, spacing): def _old_normalize(self, leaf, spacing):
value = leaf.value value = leaf.value
if value == ',' and leaf.parent.type == 'dictorsetmaker': if value == ',' and leaf.parent.type == 'dictorsetmaker':
self._indentation_stack.pop() self._indentation_tos = self._indentation_tos.parent
node = self._indentation_stack[-1] node = self._indentation_tos
if False and info.has_backslash and node.type != IndentationTypes.BACKSLASH: if False and info.has_backslash and node.type != IndentationTypes.BACKSLASH:
if node.type != IndentationTypes.SUITE: if node.type != IndentationTypes.SUITE:
@@ -327,88 +337,75 @@ class PEP8Normalizer(Normalizer):
if self._in_suite_introducer and node.type == IndentationTypes.SUITE: if self._in_suite_introducer and node.type == IndentationTypes.SUITE:
indentation += self._config.indentation indentation += self._config.indentation
node = BackslashNode( self._indentation_tos = BackslashNode(
self._config, self._config,
indentation, indentation,
leaf leaf,
) parent=self._indentation_tos
self._indentation_stack.append(node) )
if self._on_newline: if self._on_newline:
indentation = spacing.value indentation = spacing.value
if node.type == IndentationTypes.BACKSLASH \ if node.type == IndentationTypes.BACKSLASH \
and self._previous_leaf.type == 'newline': and self._previous_leaf.type == 'newline':
self._indentation_stack.pop() self._indentation_tos = self._indentation_tos.parent
if indentation != node.indentation: if not self._check_tabs_spaces(spacing):
if not self._check_tabs_spaces(spacing): should_be_indentation = node.indentation
s = '%s %s' % (len(self._config.indentation), self._indentation_type) if leaf.type == 'comment':
self.add_issue(111, 'Indentation is not a multiple of ' + s, leaf) # Comments can be dedented. So we have to care for that.
else: n = self._last_indentation_tos
if value in '])}': while True:
should_be_indentation = node.bracket_indentation if len(indentation) > len(n.indentation):
break
should_be_indentation = n.indentation
self._last_indentation_tos = n
if n == node:
break
n = n.parent
if self._new_statement:
if indentation != should_be_indentation:
s = '%s %s' % (len(self._config.indentation), self._indentation_type)
self.add_issue(111, 'Indentation is not a multiple of ' + s, leaf)
else: else:
should_be_indentation = node.indentation if value in '])}':
if self._in_suite_introducer and indentation == \ should_be_indentation = node.bracket_indentation
self._indentation_stack.get_latest_suite_node().indentation \ else:
+ self._config.indentation: should_be_indentation = node.indentation
self.add_issue(129, "Line with same indent as next logical block", leaf) if self._in_suite_introducer and indentation == \
elif indentation != should_be_indentation: self._indentation_stack.get_latest_suite_node().indentation \
if not self._check_tabs_spaces(spacing): + self._config.indentation:
if value in '])}': self.add_issue(129, "Line with same indent as next logical block", leaf)
if node.type == IndentationTypes.VERTICAL_BRACKET: elif indentation != should_be_indentation:
self.add_issue(124, "Closing bracket does not match visual indentation", leaf) if not self._check_tabs_spaces(spacing):
else: if value in '])}':
self.add_issue(123, "Losing bracket does not match indentation of opening bracket's line", leaf)
else:
if len(indentation) < len(should_be_indentation):
if node.type == IndentationTypes.VERTICAL_BRACKET: if node.type == IndentationTypes.VERTICAL_BRACKET:
self.add_issue(128, 'Continuation line under-indented for visual indent', leaf) self.add_issue(124, "Closing bracket does not match visual indentation", leaf)
elif node.type == IndentationTypes.BACKSLASH:
self.add_issue(122, 'Continuation line missing indentation or outdented', leaf)
elif node.type == IndentationTypes.IMPLICIT:
self.add_issue(135, 'xxx', leaf)
else: else:
self.add_issue(121, 'Continuation line under-indented for hanging indent', leaf) self.add_issue(123, "Losing bracket does not match indentation of opening bracket's line", leaf)
else: else:
if node.type == IndentationTypes.VERTICAL_BRACKET: if len(indentation) < len(should_be_indentation):
self.add_issue(127, 'Continuation line over-indented for visual indent', leaf) if node.type == IndentationTypes.VERTICAL_BRACKET:
elif node.type == IndentationTypes.IMPLICIT: self.add_issue(128, 'Continuation line under-indented for visual indent', leaf)
self.add_issue(136, 'xxx', leaf) elif node.type == IndentationTypes.BACKSLASH:
self.add_issue(122, 'Continuation line missing indentation or outdented', leaf)
elif node.type == IndentationTypes.IMPLICIT:
self.add_issue(135, 'xxx', leaf)
else:
self.add_issue(121, 'Continuation line under-indented for hanging indent', leaf)
else: else:
self.add_issue(126, 'Continuation line over-indented for hanging indent', leaf) if node.type == IndentationTypes.VERTICAL_BRACKET:
self.add_issue(127, 'Continuation line over-indented for visual indent', leaf)
elif node.type == IndentationTypes.IMPLICIT:
self.add_issue(136, 'xxx', leaf)
else:
self.add_issue(126, 'Continuation line over-indented for hanging indent', leaf)
else: else:
self._check_spacing(leaf, spacing) self._check_spacing(leaf, spacing)
first = True
for comment in []:#info.comments:
if first and not self._on_newline:
continue
first = False
actual_len = len(comment.indentation)
# Comments can be dedented. So we have to care for that.
for i in range(self._last_indentation_level, len(self._indentation_stack) - 2, -1):
should_be_indentation = i * self._config.indentation
should_len = len(should_be_indentation)
if actual_len >= should_len:
break
if comment.indentation == should_be_indentation:
self._last_indentation_level = i
else:
if not self._check_tabs_spaces(spacing):
if actual_len < should_len:
self.add_issue(115, 'Expected an indented block (comment)', comment)
elif actual_len > should_len:
self.add_issue(116, 'Unexpected indentation (comment)', comment)
else:
self.add_issue(114, 'indentation is not a multiple of four (comment)', comment)
self._on_newline = True
self._analyse_non_prefix(leaf) self._analyse_non_prefix(leaf)
# ------------------------------- # -------------------------------
@@ -420,39 +417,36 @@ class PEP8Normalizer(Normalizer):
# Figure out here what the indentation is. For chained brackets # Figure out here what the indentation is. For chained brackets
# we can basically use the previous indentation. # we can basically use the previous indentation.
previous_leaf = leaf previous_leaf = leaf
index = 1 n = self._indentation_tos
while not info.newline_count: while True:#not info.newline_count:
previous_leaf = previous_leaf.get_previous_leaf() previous_leaf = previous_leaf.get_previous_leaf()
n = self._indentation_stack[-index]
if not isinstance(n, BracketNode) or previous_leaf != n.leaf: if not isinstance(n, BracketNode) or previous_leaf != n.leaf:
break break
index += 1 n = n.parent
indentation = self._indentation_stack[-index].indentation indentation = n.indentation
if self._in_suite_introducer and node.type == IndentationTypes.SUITE: if self._in_suite_introducer and node.type == IndentationTypes.SUITE:
indentation += self._config.indentation indentation += self._config.indentation
self._indentation_stack.append( self._indentation_tos = BracketNode(
BracketNode( self._config, indentation, leaf,
self._config, indentation, parent=self._indentation_tos
leaf
)
) )
else: else:
assert node.type != IndentationTypes.IMPLICIT assert node.type != IndentationTypes.IMPLICIT
self._indentation_stack.pop() self._indentation_tos = self._indentation_tos.parent
elif value in ('=', ':') and self._implicit_indentation_possible \ elif value in ('=', ':') and self._implicit_indentation_possible \
and leaf.parent.type in _IMPLICIT_INDENTATION_TYPES: and leaf.parent.type in _IMPLICIT_INDENTATION_TYPES:
indentation = node.indentation indentation = node.indentation
self._indentation_stack.append( self._indentation_tos = ImplicitNode(
ImplicitNode(self._config, indentation, leaf) self._config, indentation, leaf,
parent=self._indentation_tos
) )
self._on_newline = leaf.type == 'newline' self._on_newline = leaf.type == 'newline'
# TODO does this work? with brackets and stuff? # TODO does this work? with brackets and stuff?
self._last_indentation_level = len(self._indentation_stack)
if self._on_newline and \ if self._on_newline and \
self._indentation_stack[-1].type == IndentationTypes.BACKSLASH: self._indentation_tos.type == IndentationTypes.BACKSLASH:
self._indentation_stack.pop() self._indentation_tos = self._indentation_tos.parent
if value == ':' and leaf.parent.type in _SUITE_INTRODUCERS: if value == ':' and leaf.parent.type in _SUITE_INTRODUCERS:
self._in_suite_introducer = False self._in_suite_introducer = False
@@ -475,9 +469,12 @@ class PEP8Normalizer(Normalizer):
if prev is not None and prev.type == 'error_leaf' or leaf.type == 'error_leaf': if prev is not None and prev.type == 'error_leaf' or leaf.type == 'error_leaf':
return return
type_ = leaf.type
if '\t' in spaces: if '\t' in spaces:
self.add_issue(223, 'Used tab to separate tokens', spacing) self.add_issue(223, 'Used tab to separate tokens', spacing)
elif leaf.type == 'newline': elif type_ == 'comment':
pass # TODO
elif type_ == 'newline':
add_if_spaces(291, 'Trailing whitespace', spacing) add_if_spaces(291, 'Trailing whitespace', spacing)
elif len(spaces) > 1: elif len(spaces) > 1:
self.add_issue(221, 'Multiple spaces used', spacing) self.add_issue(221, 'Multiple spaces used', spacing)
@@ -528,7 +525,7 @@ class PEP8Normalizer(Normalizer):
message_225 = 'Missing whitespace between tokens' message_225 = 'Missing whitespace between tokens'
add_not_spaces(225, message_225, spacing) add_not_spaces(225, message_225, spacing)
#print('x', leaf.start_pos, leaf, prev) #print('x', leaf.start_pos, leaf, prev)
elif leaf.type == 'keyword' or prev.type == 'keyword': elif type_ == 'keyword' or prev.type == 'keyword':
add_not_spaces(275, 'Missing whitespace around keyword', spacing) add_not_spaces(275, 'Missing whitespace around keyword', spacing)
else: else:
prev_info = self._previous_spacing prev_info = self._previous_spacing

2
parso/python/prefix.py
View File

@@ -68,7 +68,6 @@ def split_prefix(leaf, start_pos):
if not value: if not value:
break break
type_ = _types[value[0]] type_ = _types[value[0]]
print(repr(spacing), repr(value), column)
yield PrefixPart( yield PrefixPart(
leaf, type_, value, spacing, leaf, type_, value, spacing,
start_pos=(line, column + start + len(spacing)) start_pos=(line, column + start + len(spacing))
@@ -79,7 +78,6 @@ def split_prefix(leaf, start_pos):
line += 1 line += 1
column = -start column = -start
print('x', repr(value), repr(spacing))
if value: if value:
spacing = '' spacing = ''
yield PrefixPart( yield PrefixPart(

28
test/normalizer_issue_files/E11.py
View File

@@ -4,14 +4,15 @@ if x > 2:
if True: if True:
#: E111:5 #: E111:5
print print
#: E116:6 #: E111:6
# #
#: E116:2 #: E111:2
# what # what
# Comment is fine # Comment is fine
# Comment is also fine # Comment is also fine
if False: if False:
pass
print print
print print
# TODO this shouldn't actually be a 111 but an IndentationError. Please # TODO this shouldn't actually be a 111 but an IndentationError. Please
@@ -19,37 +20,40 @@ print
#: E111:4 #: E111:4
print print
mimetype = 'application/x-directory' mimetype = 'application/x-directory'
#: E116:5 #: E111:5
# 'httpd/unix-directory' # 'httpd/unix-directory'
create_date = False create_date = False
def start(self): def start(self):
# foo
#: E111:8
# bar
if True: # Hello if True: # Hello
self.master.start() # Comment self.master.start() # Comment
# try: # try:
#: E116:12 #: E111:12
# self.master.start() # self.master.start()
# except MasterExit: # except MasterExit:
#: E116:12 #: E111:12
# self.shutdown() # self.shutdown()
# finally: # finally:
#: E116:12 #: E111:12
# sys.exit() # sys.exit()
# Dedent to the first level # Dedent to the first level
#: E116:6 #: E111:6
# error # error
# Dedent to the base level # Dedent to the base level
#: E116:2 #: E111:2
# Also wrongly indented. # Also wrongly indented.
# Indent is correct. # Indent is correct.
def start(self): # Correct comment def start(self): # Correct comment
if True: if True:
#: E115:0 #: E111:0
# try: # try:
#: E115:0 #: E111:0
# self.master.start() # self.master.start()
#: E115:0 #: E111:0
# except MasterExit: # except MasterExit:
#: E115:0 #: E111:0
# self.shutdown() # self.shutdown()
self.master.start() # comment self.master.start() # comment

1
test/test_normalizer_issues_files.py
View File

@@ -5,7 +5,6 @@ tests of pydocstyle.
import difflib import difflib
import re import re
from textwrap import dedent
from functools import total_ordering from functools import total_ordering
import parso import parso