Get the indentation errors of files mostly right even in comments.

parso/python/normalizer.py

@@ -12,6 +12,13 @@ class CompressNormalizer(Normalizer):
         return leaf.prefix + leaf.value
 
 
+class Comment(object):
+    def __init__(self, comment_token, indentation):
+        self.comment_token = comment_token
+        self.indentation = indentation
+        self.start_pos = self.comment_token.start_pos
+
+
 class WhitespaceInfo(object):
     def __init__(self, leaf):
         parts = list(leaf._split_prefix())
@@ -24,11 +31,23 @@ class WhitespaceInfo(object):
     '\r': 'newline',
     '\t': 'tabs',
 '''
+        self.has_backslash = False
+        self.comments = []
+        indentation = ''
         for part in parts:
-            if part.type:
+            if part.type == 'backslash':
-                part
+                self.has_backslash = True
+
+            if part.type == 'comment':
+                self.comments.append(Comment(part, indentation))
+
+            if part.type not in ('tabs', 'spaces'):
+                indentation = ''
+            else:
+                indentation += part.value
+        self.indentation = indentation
+
         self.newline_count = 2
-        self.indentation = '  '
         self.trailing_whitespace = []
         self.comment_whitespace = []
 
@@ -40,7 +59,9 @@ def _is_magic_name(name):
 class PEP8Normalizer(Normalizer):
     def __init__(self, config):
         super(PEP8Normalizer, self).__init__(config)
-        self.indentation = 0
+        self._indentation_level = 0
+        self._last_indentation_level = 0
+        self._on_newline = True
 
     @contextmanager
     def visit_node(self, node):
@@ -103,12 +124,54 @@ class PEP8Normalizer(Normalizer):
                     break
 
         if typ == 'suite':
-            self.indentation += 1
+            self._indentation_level += 1
         yield
         if typ == 'suite':
-            self.indentation -= 1
+            self._indentation_level -= 1
 
     def normalize(self, leaf):
+        info = WhitespaceInfo(leaf)
+        should_be_indenation = self._indentation_level * self._config.indentation
+        if self._on_newline:
+            if info.indentation != should_be_indenation:
+                self.add_issue(111, 'Indentation is not a multiple of four', leaf)
+
+        first = True
+        for comment in info.comments:
+            if first and not self._on_newline:
+                    continue
+            first = False
+
+            actual_len = len(comment.indentation)
+            # Comments can be dedented. So we have to care for that.
+            for i in range(self._last_indentation_level, self._indentation_level - 1, -1):
+                should_be_indenation = i * self._config.indentation
+                should_len = len(should_be_indenation)
+                if actual_len >= should_len:
+                    break
+
+
+            if comment.indentation == should_be_indenation:
+                self._last_indentation_level = i
+            else:
+                if actual_len < should_len:
+                    self.add_issue(115, 'Expected an indented block (comment)', comment)
+                elif actual_len > should_len:
+                    self.add_issue(116, 'Unexpected indentation (comment)', comment)
+                else:
+                    self.add_issue(114, 'indentation is not a multiple of four (comment)', comment)
+
+            self._on_newline = True
+
+        self._analyse_non_prefix(leaf)
+
+        self._on_newline = leaf.type == 'newline'
+        self._last_indentation_level = self._indentation_level
+
+        return leaf.value
+
+
+    def _analyse_non_prefix(self, leaf):
         typ = leaf.type
         if typ == 'name' and leaf.value in ('l', 'O', 'I'):
             if leaf.is_definition():
@@ -156,8 +219,6 @@ class PEP8Normalizer(Normalizer):
                 else:
                     self.add_issue(714, "test for object identity should be 'is not'", leaf)
 
-        for part in leaf._split_prefix():
-            part
         return leaf.value
 
 
@@ -166,6 +227,8 @@ class PEP8NormalizerConfig(NormalizerConfig):
     """
     Normalizing to PEP8. Not really implemented, yet.
     """
+    def __init__(self):
+        self.indentation = ' ' * 4
 
 
 @PEP8NormalizerConfig.register_rule

test/normalizer_issue_files/E11.py

@@ -0,0 +1,52 @@
+if x > 2:
+    #: E111:2
+  print x
+if True:
+    #: E111:5
+     print
+    #: E116:6
+      # 
+    #: E116:2
+  # what
+    # Comment is fine
+# Comment is also fine
+
+if False:
+print
+print
+    print
+mimetype = 'application/x-directory'
+#: E116:5
+     # 'httpd/unix-directory'
+create_date = False
+
+def start(self):
+    if True: # Hello
+        self.master.start() # Comment
+        # try:
+        #: E116:12
+            # self.master.start()
+        # except MasterExit:
+        #: E116:12
+            # self.shutdown()
+        # finally:
+        #: E116:12
+            # sys.exit()
+    # Dedent to the first level
+    #: E116:6
+      # error
+# Dedent to the base level
+#: E116:2
+  # Also wrongly indented.
+# Indent is correct.
+def start(self):  # Correct comment
+    if True:
+        #: E115:0
+#       try:
+        #: E115:0
+#           self.master.start()
+        #: E115:0
+#       except MasterExit:
+        #: E115:0
+#           self.shutdown()
+        self.master.start() # comment

test/normalizer_issue_files/E72.py

@@ -1,7 +1,7 @@
-   #: E721
+#: E721:3
 if type(res) == type(42):
     pass
-   #: E721
+#: E721:3
 if type(res) != type(""):
     pass
 

test/test_normalizer_issues_files.py

@@ -37,7 +37,7 @@ def test_normalizer_issue(normalizer_issue_file):
     assert i == d, dedent("""
         Test %r failed (%s of %s passed).
         not raised  = %s
-        unspecified = %s
+        unexpected = %s
         """) % (
             normalizer_issue_file.name, len(i & d), len(d),
             sorted(d - i), sorted(i - d)