Finally fix most of the issues in E22. Huge amounts of work.

parso/python/normalizer.py

@@ -2,13 +2,23 @@ import re
 from contextlib import contextmanager
 
 from parso.normalizer import Normalizer, Rule, NormalizerConfig
+from parso.python.prefix import PrefixPart
 
 
 _IMPORT_TYPES = ('import_name', 'import_from')
 _SUITE_INTRODUCERS = ('classdef', 'funcdef', 'if_stmt', 'while_stmt',
                       'for_stmt', 'try_stmt', 'with_stmt')
+_NON_STAR_TYPES = ('term', 'import_from', 'power')
 _OPENING_BRACKETS = '(', '[', '{'
 _CLOSING_BRACKETS = ')', ']', '}'
+# TODO ~ << >> & | ^
+_FACTOR = '+', '-', '~'
+_ALLOW_SPACE = '*', '+', '-', '**', '/', '//', '@'
+_BITWISE_OPERATOR = '<<', '>>', '|', '&', '^'
+_NEEDS_SPACE = '=', '<', '>', '==', '>=', '<=', '<>', '!=', '%', \
+               '+=', '-=', '*=', '@=', '/=', '%=', '&=', '|=', '^=', '<<=', \
+               '>>=', '**=', '//='
+_NEEDS_SPACE += _BITWISE_OPERATOR
 _IMPLICIT_INDENTATION_TYPES = ('dictorsetmaker', 'argument')
 
 class CompressNormalizer(Normalizer):
@@ -44,7 +54,14 @@ class WhitespaceInfo(object):
 '''
         self.has_backslash = False
         self.comments = []
-        indentation_part = None
+        # TODO this should probably be moved to a function that gets the
+        # indentation part.
+        if parts:
+            start_pos = parts[0].start_pos
+        else:
+            start_pos = leaf.start_pos
+        indentation_part = PrefixPart(leaf, 'indentation', '', start_pos)
+
         self.newline_count = 0
         for part in parts:
             if part.type == 'backslash':
@@ -339,21 +356,7 @@ class PEP8Normalizer(Normalizer):
                                 else:
                                     self.add_issue(126, 'Continuation line over-indented for hanging indent', leaf)
         else:
-            spaces = info.indentation
+            self._check_spacing(leaf, info)
-            if spaces:
-                if leaf in _CLOSING_BRACKETS:
-                    message = "Whitespace before '%s'" % leaf.value
-                    self.add_issue(202, message, info.indentation_part)
-                elif leaf in (',', ';') or leaf == ':' \
-                        and leaf.parent.type not in ('subscript', 'subscriptlist'):
-                    message = "Whitespace before '%s'" % leaf.value
-                    self.add_issue(203, message, info.indentation_part)
-                elif leaf in _OPENING_BRACKETS:
-                    message = "Whitespace before '%s'" % leaf.value
-                    self.add_issue(211, message, info.indentation_part)
-                elif self._previous_leaf in _OPENING_BRACKETS:
-                    message = "Whitespace after '%s'" % leaf.value
-                    self.add_issue(201, message, info.indentation_part)
 
         first = True
         for comment in info.comments:
@@ -432,8 +435,86 @@ class PEP8Normalizer(Normalizer):
             self._in_suite_introducer = False
 
         self._previous_leaf = leaf
+        self._previous_whitespace_info = info
         return value
 
+    def _check_spacing(self, leaf, info):
+        spaces = info.indentation
+        prev = self._previous_leaf
+        if '\t' in spaces:
+            self.add_issue(223, 'Used tab to separate tokens', info.indentation_part)
+        elif len(spaces) > 1:
+            self.add_issue(221, 'Multiple spaces used', info.indentation_part)
+        elif info.comments:
+            pass
+        else:
+            def add_if_spaces(*args):
+                if spaces:
+                    return self.add_issue(*args)
+
+            def add_not_spaces(*args):
+                if not spaces:
+                    return self.add_issue(*args)
+
+            if leaf.type == 'newline':
+                add_if_spaces(291, 'Trailing whitespace', info.indentation_part)
+            elif prev in _OPENING_BRACKETS:
+                message = "Whitespace after '%s'" % leaf.value
+                add_if_spaces(201, message, info.indentation_part)
+            elif leaf in _CLOSING_BRACKETS:
+                message = "Whitespace before '%s'" % leaf.value
+                add_if_spaces(202, message, info.indentation_part)
+            #elif leaf in _OPENING_BRACKETS:
+                # TODO
+            #    if False:
+            #        message = "Whitespace before '%s'" % leaf.value
+            #        add_if_spaces(211, message, info.indentation_part)
+            elif leaf in (',', ';') or leaf == ':' \
+                    and leaf.parent.type not in ('subscript', 'subscriptlist'):
+                message = "Whitespace before '%s'" % leaf.value
+                add_if_spaces(203, message, info.indentation_part)
+            elif leaf == ':':  # Is a subscript
+                # TODO
+                pass
+            elif prev.type == 'keyword':
+                add_not_spaces(275, 'Missing whitespace around keyword', info.indentation_part)
+            elif leaf.type == 'keyword':
+                add_not_spaces(275, 'Missing whitespace around keyword', info.indentation_part)
+            elif prev in (',', ';', ':'):
+                # TODO
+                pass
+            elif leaf in ('*', '**') and leaf.parent.type not in _NON_STAR_TYPES \
+                    or prev in ('*', '**') \
+                    and prev.parent.type not in _NON_STAR_TYPES:
+                # TODO
+                pass
+            elif prev in _FACTOR and prev.parent.type == 'factor':
+                pass
+            elif leaf in _NEEDS_SPACE or prev in _NEEDS_SPACE:
+                if leaf == '=' and leaf.parent.type in ('argument', 'param') \
+                        or prev == '=' and prev.parent.type in ('argument', 'param'):
+                    add_if_spaces(251, 'Unexpected spaces around keyword / parameter equals', info.indentation_part)
+                elif leaf in _BITWISE_OPERATOR or prev in _BITWISE_OPERATOR:
+                    add_not_spaces(227, 'Missing whitespace around bitwise or shift operator', info.indentation_part)
+                elif leaf == '%' or prev == '%':
+                    add_not_spaces(228, 'Missing whitespace around modulo operator', info.indentation_part)
+                else:
+                    message_225 = 'Missing whitespace between tokens'
+                    add_not_spaces(225, message_225, info.indentation_part)
+                    #print('x', leaf.start_pos, leaf, prev)
+            else:
+                prev_info = self._previous_whitespace_info
+                message_225 = 'Missing whitespace between tokens'
+                if prev in _ALLOW_SPACE and spaces != prev_info.indentation:
+                    message = "Whitespace before operator doesn't match with whitespace after"
+                    self.add_issue(229, message, info.indentation_part)
+
+                if spaces and leaf not in _ALLOW_SPACE and prev not in _ALLOW_SPACE:
+                    #print(leaf, prev)
+                    self.add_issue(225, message_225, info.indentation_part)
+
+                #if not prev_info.indentation and leaf not in _ALLOW_SPACE:
+                    #self.add_issue(225, message_225, prev_info.indentation_part)
 
     def _analyse_non_prefix(self, leaf):
         typ = leaf.type
@@ -492,16 +573,10 @@ class PEP8Normalizer(Normalizer):
         return leaf.value
 
     def add_issue(self, code, message, node):
-        try:
+        from parso.python.tree import search_ancestor
-            parent = node.parent
+        if search_ancestor(node, 'error_node') is not None or \
-        except AttributeError:
+                search_ancestor(self._previous_leaf, 'error_node') is not None:
-            # TODO for prefix parts, there's no parents yet.
+            return
-            pass
-        else:
-            while parent:
-                if parent.type == 'error_node':
-                    return
-                parent = parent.parent
         super(PEP8Normalizer, self).add_issue(code, message, node)
 
 

parso/python/prefix.py

@@ -4,7 +4,8 @@ from parso.tokenize import group
 
 
 class PrefixPart(object):
-    def __init__(self, typ, value, start_pos):
+    def __init__(self, leaf, typ, value, start_pos):
+        self.parent = leaf
         self.type = typ
         self.value = value
         self.start_pos = start_pos
@@ -45,14 +46,14 @@ _types = {
 }
 
 
-def split_prefix(prefix, start_pos):
+def split_prefix(leaf, start_pos):
     line, column = start_pos
     start = 0
-    while start != len(prefix):
+    while start != len(leaf.prefix):
-        match =_regex.match(prefix, start)
+        match =_regex.match(leaf.prefix, start)
         value = match.group(0)
         typ = _types[value[0]]
-        yield PrefixPart(typ, value, (line, column + start))
+        yield PrefixPart(leaf, typ, value, (line, column + start))
 
         start = match.end(0)
         if value.endswith('\n'):

parso/python/tree.py

@@ -101,7 +101,7 @@ class PythonLeaf(PythonMixin, Leaf):
     __slots__ = ()
 
     def _split_prefix(self):
-        return split_prefix(self.prefix, self.get_start_pos_of_prefix())
+        return split_prefix(self, self.get_start_pos_of_prefix())
 
 
 class _LeafWithoutNewlines(PythonLeaf):

test/normalizer_issue_files/E22.py

@@ -0,0 +1,158 @@
+a = 12 + 3
+#: E221:5 E229:8
+b = 4  + 5
+#: E221:1
+x             = 1
+#: E221:1
+y             = 2
+long_variable = 3
+#: E221:4
+x[0]          = 1
+#: E221:4
+x[1]          = 2
+long_variable = 3
+#: E221:8 E229:19
+x = f(x)          + 1
+y = long_variable + 2
+#: E221:8 E229:19
+z = x[0]          + 3
+#: E221+2:13
+text = """
+    bar
+    foo %s"""  % rofl
+# Okay
+x = 1
+y = 2
+long_variable = 3
+
+
+#: E221:7
+a = a +  1
+b = b + 10
+#: E221:3
+x =            -1
+#: E221:3
+y =            -2
+long_variable = 3
+#: E221:6
+x[0] =          1
+#: E221:6
+x[1] =          2
+long_variable = 3
+
+
+#: E223+1:1
+foobart = 4
+a	= 3  # aligned with tab
+
+
+#: E223:4
+a +=	1
+b += 1000
+
+
+#: E225:12
+submitted +=1
+#: E225:9
+submitted+= 1
+#: E225:3
+c =-1
+#: E229:7
+x = x /2 - 1
+#: E229:11
+c = alpha -4
+#: E229:10
+c = alpha- 4
+#: E229:8
+z = x **y
+#: E229:14
+z = (x + 1) **y
+#: E229:13
+z = (x + 1)** y
+#: E227:14
+_1kB = _1MB >>10
+#: E227:11
+_1kB = _1MB>> 10
+#: E225:1 E225:2 E229:4
+i=i+ 1
+#: E225:1 E225:2 E229:5
+i=i +1
+#: E225:1 E225:2
+i=i+1
+#: E225:3
+i =i+1
+#: E225:1
+i= i+1
+#: E229:8
+c = (a +b)*(a - b)
+#: E229:7
+c = (a+ b)*(a - b)
+
+z = 2//30
+c = (a+b) * (a-b)
+#: E275:13 E275:14
+norman = True+False
+x = x*2 - 1
+x = x/2 - 1
+#: E999
+x = x / 2-1
+
+hypot2 = x*x + y*y
+c = (a + b)*(a - b)
+
+def halves(n):
+    return (i//2 for i in range(n))
+#: E227:11 E227:13
+_1kB = _1MB>>10
+#: E227:11 E227:13
+_1MB = _1kB<<10
+#: E227:5 E227:6
+a = b|c
+#: E227:5 E227:6
+b = c&a
+#: E227:5 E227:6
+c = b^a
+#: E228:5 E228:6
+a = b%c
+#: E228:9 E228:10
+msg = fmt%(errno, errmsg)
+#: E228:25 E228:26
+msg = "Error %d occurred"%errno
+
+#: E228:7
+a = b %c
+a = b % c
+
+# Okay
+i = i + 1
+submitted += 1
+x = x * 2 - 1
+hypot2 = x * x + y * y
+c = (a + b) * (a - b)
+_1MiB = 2 ** 20
+_1TiB = 2**30
+foo(bar, key='word', *args, **kwargs)
+baz(**kwargs)
+negative = -1
+spam(-1)
+-negative
+func1(lambda *args, **kw: (args, kw))
+func2(lambda a, b=h[:], c=0: (a, b, c))
+if not -5 < x < +5:
+    #: E227:12
+    print >>sys.stderr, "x is out of range."
+print >> sys.stdout, "x is an integer."
+x = x / 2 - 1
+
+if alpha[:-i]:
+    *a, b = (1, 2, 3)
+
+
+def squares(n):
+    return (i**2 for i in range(n))
+
+
+ENG_PREFIXES = {
+    -6: "\u03bc",  # Greek letter mu
+    -3: "m",
+}