From a341af1f81a1c10ea51ac7aa117162d7ceebe8a6 Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Tue, 27 Jun 2017 09:55:18 +0200 Subject: [PATCH] Finally fix most of the issues in E22. Huge amounts of work. --- parso/python/normalizer.py | 127 ++++++++++++++++++----- parso/python/prefix.py | 11 +- parso/python/tree.py | 2 +- test/normalizer_issue_files/E22.py | 158 +++++++++++++++++++++++++++++ 4 files changed, 266 insertions(+), 32 deletions(-) create mode 100644 test/normalizer_issue_files/E22.py diff --git a/parso/python/normalizer.py b/parso/python/normalizer.py index 0ce3e88..55b3a23 100644 --- a/parso/python/normalizer.py +++ b/parso/python/normalizer.py @@ -2,13 +2,23 @@ import re from contextlib import contextmanager from parso.normalizer import Normalizer, Rule, NormalizerConfig +from parso.python.prefix import PrefixPart _IMPORT_TYPES = ('import_name', 'import_from') _SUITE_INTRODUCERS = ('classdef', 'funcdef', 'if_stmt', 'while_stmt', 'for_stmt', 'try_stmt', 'with_stmt') +_NON_STAR_TYPES = ('term', 'import_from', 'power') _OPENING_BRACKETS = '(', '[', '{' _CLOSING_BRACKETS = ')', ']', '}' +# TODO ~ << >> & | ^ +_FACTOR = '+', '-', '~' +_ALLOW_SPACE = '*', '+', '-', '**', '/', '//', '@' +_BITWISE_OPERATOR = '<<', '>>', '|', '&', '^' +_NEEDS_SPACE = '=', '<', '>', '==', '>=', '<=', '<>', '!=', '%', \ + '+=', '-=', '*=', '@=', '/=', '%=', '&=', '|=', '^=', '<<=', \ + '>>=', '**=', '//=' +_NEEDS_SPACE += _BITWISE_OPERATOR _IMPLICIT_INDENTATION_TYPES = ('dictorsetmaker', 'argument') class CompressNormalizer(Normalizer): @@ -44,7 +54,14 @@ class WhitespaceInfo(object): ''' self.has_backslash = False self.comments = [] - indentation_part = None + # TODO this should probably be moved to a function that gets the + # indentation part. + if parts: + start_pos = parts[0].start_pos + else: + start_pos = leaf.start_pos + indentation_part = PrefixPart(leaf, 'indentation', '', start_pos) + self.newline_count = 0 for part in parts: if part.type == 'backslash': @@ -339,21 +356,7 @@ class PEP8Normalizer(Normalizer): else: self.add_issue(126, 'Continuation line over-indented for hanging indent', leaf) else: - spaces = info.indentation - if spaces: - if leaf in _CLOSING_BRACKETS: - message = "Whitespace before '%s'" % leaf.value - self.add_issue(202, message, info.indentation_part) - elif leaf in (',', ';') or leaf == ':' \ - and leaf.parent.type not in ('subscript', 'subscriptlist'): - message = "Whitespace before '%s'" % leaf.value - self.add_issue(203, message, info.indentation_part) - elif leaf in _OPENING_BRACKETS: - message = "Whitespace before '%s'" % leaf.value - self.add_issue(211, message, info.indentation_part) - elif self._previous_leaf in _OPENING_BRACKETS: - message = "Whitespace after '%s'" % leaf.value - self.add_issue(201, message, info.indentation_part) + self._check_spacing(leaf, info) first = True for comment in info.comments: @@ -432,8 +435,86 @@ class PEP8Normalizer(Normalizer): self._in_suite_introducer = False self._previous_leaf = leaf + self._previous_whitespace_info = info return value + def _check_spacing(self, leaf, info): + spaces = info.indentation + prev = self._previous_leaf + if '\t' in spaces: + self.add_issue(223, 'Used tab to separate tokens', info.indentation_part) + elif len(spaces) > 1: + self.add_issue(221, 'Multiple spaces used', info.indentation_part) + elif info.comments: + pass + else: + def add_if_spaces(*args): + if spaces: + return self.add_issue(*args) + + def add_not_spaces(*args): + if not spaces: + return self.add_issue(*args) + + if leaf.type == 'newline': + add_if_spaces(291, 'Trailing whitespace', info.indentation_part) + elif prev in _OPENING_BRACKETS: + message = "Whitespace after '%s'" % leaf.value + add_if_spaces(201, message, info.indentation_part) + elif leaf in _CLOSING_BRACKETS: + message = "Whitespace before '%s'" % leaf.value + add_if_spaces(202, message, info.indentation_part) + #elif leaf in _OPENING_BRACKETS: + # TODO + # if False: + # message = "Whitespace before '%s'" % leaf.value + # add_if_spaces(211, message, info.indentation_part) + elif leaf in (',', ';') or leaf == ':' \ + and leaf.parent.type not in ('subscript', 'subscriptlist'): + message = "Whitespace before '%s'" % leaf.value + add_if_spaces(203, message, info.indentation_part) + elif leaf == ':': # Is a subscript + # TODO + pass + elif prev.type == 'keyword': + add_not_spaces(275, 'Missing whitespace around keyword', info.indentation_part) + elif leaf.type == 'keyword': + add_not_spaces(275, 'Missing whitespace around keyword', info.indentation_part) + elif prev in (',', ';', ':'): + # TODO + pass + elif leaf in ('*', '**') and leaf.parent.type not in _NON_STAR_TYPES \ + or prev in ('*', '**') \ + and prev.parent.type not in _NON_STAR_TYPES: + # TODO + pass + elif prev in _FACTOR and prev.parent.type == 'factor': + pass + elif leaf in _NEEDS_SPACE or prev in _NEEDS_SPACE: + if leaf == '=' and leaf.parent.type in ('argument', 'param') \ + or prev == '=' and prev.parent.type in ('argument', 'param'): + add_if_spaces(251, 'Unexpected spaces around keyword / parameter equals', info.indentation_part) + elif leaf in _BITWISE_OPERATOR or prev in _BITWISE_OPERATOR: + add_not_spaces(227, 'Missing whitespace around bitwise or shift operator', info.indentation_part) + elif leaf == '%' or prev == '%': + add_not_spaces(228, 'Missing whitespace around modulo operator', info.indentation_part) + else: + message_225 = 'Missing whitespace between tokens' + add_not_spaces(225, message_225, info.indentation_part) + #print('x', leaf.start_pos, leaf, prev) + else: + prev_info = self._previous_whitespace_info + message_225 = 'Missing whitespace between tokens' + if prev in _ALLOW_SPACE and spaces != prev_info.indentation: + message = "Whitespace before operator doesn't match with whitespace after" + self.add_issue(229, message, info.indentation_part) + + if spaces and leaf not in _ALLOW_SPACE and prev not in _ALLOW_SPACE: + #print(leaf, prev) + self.add_issue(225, message_225, info.indentation_part) + + #if not prev_info.indentation and leaf not in _ALLOW_SPACE: + #self.add_issue(225, message_225, prev_info.indentation_part) def _analyse_non_prefix(self, leaf): typ = leaf.type @@ -492,16 +573,10 @@ class PEP8Normalizer(Normalizer): return leaf.value def add_issue(self, code, message, node): - try: - parent = node.parent - except AttributeError: - # TODO for prefix parts, there's no parents yet. - pass - else: - while parent: - if parent.type == 'error_node': - return - parent = parent.parent + from parso.python.tree import search_ancestor + if search_ancestor(node, 'error_node') is not None or \ + search_ancestor(self._previous_leaf, 'error_node') is not None: + return super(PEP8Normalizer, self).add_issue(code, message, node) diff --git a/parso/python/prefix.py b/parso/python/prefix.py index 5757d6f..5260760 100644 --- a/parso/python/prefix.py +++ b/parso/python/prefix.py @@ -4,7 +4,8 @@ from parso.tokenize import group class PrefixPart(object): - def __init__(self, typ, value, start_pos): + def __init__(self, leaf, typ, value, start_pos): + self.parent = leaf self.type = typ self.value = value self.start_pos = start_pos @@ -45,14 +46,14 @@ _types = { } -def split_prefix(prefix, start_pos): +def split_prefix(leaf, start_pos): line, column = start_pos start = 0 - while start != len(prefix): - match =_regex.match(prefix, start) + while start != len(leaf.prefix): + match =_regex.match(leaf.prefix, start) value = match.group(0) typ = _types[value[0]] - yield PrefixPart(typ, value, (line, column + start)) + yield PrefixPart(leaf, typ, value, (line, column + start)) start = match.end(0) if value.endswith('\n'): diff --git a/parso/python/tree.py b/parso/python/tree.py index c11921c..8490ba2 100644 --- a/parso/python/tree.py +++ b/parso/python/tree.py @@ -101,7 +101,7 @@ class PythonLeaf(PythonMixin, Leaf): __slots__ = () def _split_prefix(self): - return split_prefix(self.prefix, self.get_start_pos_of_prefix()) + return split_prefix(self, self.get_start_pos_of_prefix()) class _LeafWithoutNewlines(PythonLeaf): diff --git a/test/normalizer_issue_files/E22.py b/test/normalizer_issue_files/E22.py new file mode 100644 index 0000000..c81e46b --- /dev/null +++ b/test/normalizer_issue_files/E22.py @@ -0,0 +1,158 @@ +a = 12 + 3 +#: E221:5 E229:8 +b = 4 + 5 +#: E221:1 +x = 1 +#: E221:1 +y = 2 +long_variable = 3 +#: E221:4 +x[0] = 1 +#: E221:4 +x[1] = 2 +long_variable = 3 +#: E221:8 E229:19 +x = f(x) + 1 +y = long_variable + 2 +#: E221:8 E229:19 +z = x[0] + 3 +#: E221+2:13 +text = """ + bar + foo %s""" % rofl +# Okay +x = 1 +y = 2 +long_variable = 3 + + +#: E221:7 +a = a + 1 +b = b + 10 +#: E221:3 +x = -1 +#: E221:3 +y = -2 +long_variable = 3 +#: E221:6 +x[0] = 1 +#: E221:6 +x[1] = 2 +long_variable = 3 + + +#: E223+1:1 +foobart = 4 +a = 3 # aligned with tab + + +#: E223:4 +a += 1 +b += 1000 + + +#: E225:12 +submitted +=1 +#: E225:9 +submitted+= 1 +#: E225:3 +c =-1 +#: E229:7 +x = x /2 - 1 +#: E229:11 +c = alpha -4 +#: E229:10 +c = alpha- 4 +#: E229:8 +z = x **y +#: E229:14 +z = (x + 1) **y +#: E229:13 +z = (x + 1)** y +#: E227:14 +_1kB = _1MB >>10 +#: E227:11 +_1kB = _1MB>> 10 +#: E225:1 E225:2 E229:4 +i=i+ 1 +#: E225:1 E225:2 E229:5 +i=i +1 +#: E225:1 E225:2 +i=i+1 +#: E225:3 +i =i+1 +#: E225:1 +i= i+1 +#: E229:8 +c = (a +b)*(a - b) +#: E229:7 +c = (a+ b)*(a - b) + +z = 2//30 +c = (a+b) * (a-b) +#: E275:13 E275:14 +norman = True+False +x = x*2 - 1 +x = x/2 - 1 +#: E999 +x = x / 2-1 + +hypot2 = x*x + y*y +c = (a + b)*(a - b) + +def halves(n): + return (i//2 for i in range(n)) +#: E227:11 E227:13 +_1kB = _1MB>>10 +#: E227:11 E227:13 +_1MB = _1kB<<10 +#: E227:5 E227:6 +a = b|c +#: E227:5 E227:6 +b = c&a +#: E227:5 E227:6 +c = b^a +#: E228:5 E228:6 +a = b%c +#: E228:9 E228:10 +msg = fmt%(errno, errmsg) +#: E228:25 E228:26 +msg = "Error %d occurred"%errno + +#: E228:7 +a = b %c +a = b % c + +# Okay +i = i + 1 +submitted += 1 +x = x * 2 - 1 +hypot2 = x * x + y * y +c = (a + b) * (a - b) +_1MiB = 2 ** 20 +_1TiB = 2**30 +foo(bar, key='word', *args, **kwargs) +baz(**kwargs) +negative = -1 +spam(-1) +-negative +func1(lambda *args, **kw: (args, kw)) +func2(lambda a, b=h[:], c=0: (a, b, c)) +if not -5 < x < +5: + #: E227:12 + print >>sys.stderr, "x is out of range." +print >> sys.stdout, "x is an integer." +x = x / 2 - 1 + +if alpha[:-i]: + *a, b = (1, 2, 3) + + +def squares(n): + return (i**2 for i in range(n)) + + +ENG_PREFIXES = { + -6: "\u03bc", # Greek letter mu + -3: "m", +}