From b3923e65e8bbce9f3d409d80ce9eb1c42165da6a Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Mon, 10 Jul 2017 23:04:54 +0200 Subject: [PATCH] Try to add syntax errors to pep8 normalizing. --- parso/normalizer.py | 4 ++ parso/python/normalizer.py | 60 ++++++++++++++----- parso/python/pep8.py | 12 +++- test/normalizer_issue_files/E10.py | 4 +- test/normalizer_issue_files/E11.py | 1 + test/normalizer_issue_files/E12_first.py | 20 +++---- test/normalizer_issue_files/E12_not_first.py | 20 +++---- test/normalizer_issue_files/E12_not_second.py | 7 +-- test/normalizer_issue_files/E12_second.py | 3 +- test/normalizer_issue_files/E12_third.py | 2 +- test/normalizer_issue_files/E20.py | 6 +- test/normalizer_issue_files/E27.py | 1 + test/test_python_errors.py | 2 +- 13 files changed, 93 insertions(+), 49 deletions(-) diff --git a/parso/normalizer.py b/parso/normalizer.py index 825fd60..28cafbd 100644 --- a/parso/normalizer.py +++ b/parso/normalizer.py @@ -7,6 +7,7 @@ class Normalizer(object): self.issues = [] def walk(self, node): + self.initialize(node) value = self.visit(node) self.finalize() return value @@ -27,6 +28,9 @@ class Normalizer(object): def visit_leaf(self, leaf): return leaf.prefix + leaf.value + def initialize(self, node): + pass + def finalize(self): pass diff --git a/parso/python/normalizer.py b/parso/python/normalizer.py index 4655fda..bac0ce7 100644 --- a/parso/python/normalizer.py +++ b/parso/python/normalizer.py @@ -1,7 +1,10 @@ from contextlib import contextmanager -from parso.normalizer import Normalizer, NormalizerConfig +from parso.normalizer import Normalizer, NormalizerConfig, Issue +_BLOCK_STMTS = ('if_stmt', 'while_stmt', 'for_stmt', 'try_stmt', 'with_stmt') +# This is the maximal block size given by python. +_MAX_BLOCK_SIZE = 20 class CompressNormalizer(Normalizer): """ @@ -11,6 +14,23 @@ class CompressNormalizer(Normalizer): return leaf.prefix + leaf.value +class Context(object): + def __init__(self, scope, parent_context=None): + self.blocks = [] + + @contextmanager + def add_block(self, node): + self.blocks.append(node) + yield + self.blocks.pop() + + @contextmanager + def add_context(self, node): + self.blocks.append(node) + yield Context(node, parent_context=self) + self.blocks.pop() + + class ErrorFinder(Normalizer): """ Searches for errors in the syntax tree. @@ -19,37 +39,49 @@ class ErrorFinder(Normalizer): super(ErrorFinder, self).__init__(*args, **kwargs) self._error_dict = {} + def initialize(self, node): + from parso.python.tree import search_ancestor + parent_scope = search_ancestor(node, 'classdef', 'funcdef', 'file_input') + self._context = Context(parent_scope) + @contextmanager def visit_node(self, node): if node.type == 'error_node': leaf = node.get_next_leaf() - self._add_error(901, "Syntax Error", leaf) + self._add_syntax_error("Syntax Error", leaf) + elif node.type in _BLOCK_STMTS: + with self._context.add_block(node): + yield + return + elif node.type in ('classdef', 'funcdef'): + context = self._context + with self._context.add_context(node) as new_context: + if len(context.blocks) == _MAX_BLOCK_SIZE: + self._add_syntax_error("Too many statically nested blocks", node) + self._context = new_context + yield + self._context = context + return yield def visit_leaf(self, leaf): if leaf.type == 'error_leaf': - self._add_error(901, "Syntax Error", leaf) + self._add_syntax_error("Syntax Error", leaf) return '' + def _add_syntax_error(self, message, node): + self._add_error(901, message, node) + def _add_error(self, code, message, node): + # Check if the issues are on the same line. line = node.start_pos[0] self._error_dict.setdefault(line, (code, message, node)) def finalize(self): for code, message, node in self._error_dict.values(): - self.add_issue(code, message, node) - - def add_issue(self, code, message, node): - # Check if the issues are on the same line. - prev = node.get_previous_leaf() - if prev is not None and prev.type == 'error_leaf': - # There's already an error nearby. There's a huge chance they are - # related, so don't report this one. - return - - super(ErrorFinder, self).add_issue(code, message, node) + self.issues.append(Issue(node, code, message)) class ErrorFinderConfig(NormalizerConfig): diff --git a/parso/python/pep8.py b/parso/python/pep8.py index 3ea7174..e959490 100644 --- a/parso/python/pep8.py +++ b/parso/python/pep8.py @@ -1,7 +1,8 @@ import re from contextlib import contextmanager -from parso.normalizer import Normalizer, Rule, NormalizerConfig +from parso.python.normalizer import ErrorFinder, ErrorFinderConfig +from parso.normalizer import Rule _IMPORT_TYPES = ('import_name', 'import_from') @@ -147,7 +148,7 @@ def _is_magic_name(name): return name.value.startswith('__') and name.value.startswith('__') -class PEP8Normalizer(Normalizer): +class PEP8Normalizer(ErrorFinder): def __init__(self, config): super(PEP8Normalizer, self).__init__(config) self._previous_part = None @@ -172,6 +173,10 @@ class PEP8Normalizer(Normalizer): @contextmanager def visit_node(self, node): + with super(PEP8Normalizer, self).visit_node(node): + return self._visit_node(node) + + def _visit_node(self, node): typ = node.type if typ in 'import_name': @@ -329,6 +334,7 @@ class PEP8Normalizer(Normalizer): self._newline_count = 0 def visit_leaf(self, leaf): + super(PEP8Normalizer, self).visit_leaf(leaf) for part in leaf._split_prefix(): if part.type == 'spacing': # This part is used for the part call after for. @@ -684,7 +690,7 @@ class PEP8Normalizer(Normalizer): super(PEP8Normalizer, self).add_issue(code, message, node) -class PEP8NormalizerConfig(NormalizerConfig): +class PEP8NormalizerConfig(ErrorFinderConfig): normalizer_class = PEP8Normalizer """ Normalizing to PEP8. Not really implemented, yet. diff --git a/test/normalizer_issue_files/E10.py b/test/normalizer_issue_files/E10.py index 2dbed78..673a615 100644 --- a/test/normalizer_issue_files/E10.py +++ b/test/normalizer_issue_files/E10.py @@ -1,7 +1,7 @@ for a in 'abc': for b in 'xyz': - print a # indented with 8 spaces - # TODO currently not an error, because the indentation matches. + print(a) # indented with 8 spaces + #: E901:1 print(b) # indented with 1 tab if True: #: E101:0 diff --git a/test/normalizer_issue_files/E11.py b/test/normalizer_issue_files/E11.py index 4bff693..ead0513 100644 --- a/test/normalizer_issue_files/E11.py +++ b/test/normalizer_issue_files/E11.py @@ -15,6 +15,7 @@ if False: pass print print +#: E901:4 print mimetype = 'application/x-directory' #: E111:5 diff --git a/test/normalizer_issue_files/E12_first.py b/test/normalizer_issue_files/E12_first.py index 49588d7..8dc65a5 100644 --- a/test/normalizer_issue_files/E12_first.py +++ b/test/normalizer_issue_files/E12_first.py @@ -1,7 +1,7 @@ -print "E121", ( +abc = "E121", ( #: E121:2 "dent") -print "E122", ( +abc = "E122", ( #: E121:0 "dent") my_list = [ @@ -9,11 +9,11 @@ my_list = [ 4, 5, 6, #: E123 ] -print "E124", ("visual", +abc = "E124", ("visual", "indent_two" #: E124:14 ) -print "E124", ("visual", +abc = "E124", ("visual", "indent_five" #: E124:0 ) @@ -25,19 +25,19 @@ if (row < 0 or self.moduleCount <= row or col < 0 or self.moduleCount <= col): raise Exception("%s,%s - %s" % (row, col, self.moduleCount)) -print "E126", ( +abc = "E126", ( #: E126:12 "dent") -print "E126", ( +abc = "E126", ( #: E126:8 "dent") -print "E127", ("over-", +abc = "E127", ("over-", #: E127:18 "over-indent") -print "E128", ("visual", +abc = "E128", ("visual", #: E128:4 "hanging") -print "E128", ("under-", +abc = "E128", ("under-", #: E128:14 "under-indent") @@ -63,7 +63,7 @@ rv.update(dict.fromkeys(( abricot = 3 + \ 4 + \ 5 + 6 -print "hello", ( +abc = "hello", ( "there", #: E126:5 diff --git a/test/normalizer_issue_files/E12_not_first.py b/test/normalizer_issue_files/E12_not_first.py index 6f6daa7..6d13b29 100644 --- a/test/normalizer_issue_files/E12_not_first.py +++ b/test/normalizer_issue_files/E12_not_first.py @@ -55,32 +55,32 @@ if start[1] > end_col and not ( "indented for visual indent") -print "OK", ("visual", +abc = "OK", ("visual", "indent") -print "Okay", ("visual", +abc = "Okay", ("visual", "indent_three" ) -print "a-ok", ( +abc = "a-ok", ( "there", "dude", ) -print "hello", ( +abc = "hello", ( "there", "dude") -print "hello", ( +abc = "hello", ( "there", # "john", "dude") -print "hello", ( +abc = "hello", ( "there", "dude") -print "hello", ( +abc = "hello", ( "there", "dude", ) @@ -196,12 +196,12 @@ if bar: if ((foo.bar("baz") and foo.bar("frop") )): - print "yes" + print("yes") # also ok, but starting to look like LISP if ((foo.bar("baz") and foo.bar("frop"))): - print "yes" + print("yes") #: E129+1:4 E127+2:9 if (a == 2 or @@ -223,7 +223,7 @@ if length > options.max_line_length: # blub -print 'l.{line}\t{pos}\t{name}\t{text}'.format( +asd = 'l.{line}\t{pos}\t{name}\t{text}'.format( line=token[2][0], pos=pos, name=tokenize.tok_name[token[0]], diff --git a/test/normalizer_issue_files/E12_not_second.py b/test/normalizer_issue_files/E12_not_second.py index 3169fca..1d4e830 100644 --- a/test/normalizer_issue_files/E12_not_second.py +++ b/test/normalizer_issue_files/E12_not_second.py @@ -83,9 +83,6 @@ help = "print total number of errors " \ help = u"print total number of errors " \ u"to standard error" -help = ur"print total number of errors " \ - ur"to standard error" - help = b"print total number of errors " \ b"to standard error" @@ -187,6 +184,8 @@ try: %s -d5 ''' % sys.argv[0]) +# The try statement above was not finished. +#: E901 d = { # comment 1: 2 } @@ -284,7 +283,7 @@ some_hash = { } -print dedent( +abc = dedent( ''' mkdir -p ./{build}/ mv ./build/ ./{build}/%(revision)s/ diff --git a/test/normalizer_issue_files/E12_second.py b/test/normalizer_issue_files/E12_second.py index 6370a3a..01f6fa3 100644 --- a/test/normalizer_issue_files/E12_second.py +++ b/test/normalizer_issue_files/E12_second.py @@ -23,6 +23,7 @@ if True: or another_very_long_variable_name: raise Exception() +#: E901+1:8 E901+5 dictionary = [ "is": { # Might be a E122:4, but is not because the code is invalid Python. @@ -40,7 +41,7 @@ setup('', #: E123+2:4 E291:15 -print "E123", ( +abc = "E123", ( "bad", "hanging", "close" ) diff --git a/test/normalizer_issue_files/E12_third.py b/test/normalizer_issue_files/E12_third.py index f0c05aa..7c6c8b1 100644 --- a/test/normalizer_issue_files/E12_third.py +++ b/test/normalizer_issue_files/E12_third.py @@ -97,7 +97,7 @@ for foo in """ 123 """.strip().split(): print(foo) -print dedent( +abc = dedent( ''' mkdir -p ./{build}/ mv ./build/ ./{build}/%(revision)s/ diff --git a/test/normalizer_issue_files/E20.py b/test/normalizer_issue_files/E20.py index 8de71f7..03d4f8b 100644 --- a/test/normalizer_issue_files/E20.py +++ b/test/normalizer_issue_files/E20.py @@ -35,18 +35,18 @@ result = [ #: E203:9 if x == 4 : - print x, y + print(x, y) x, y = y, x if x == 4: #: E203:12 E702:13 a = x, y ; x, y = y, x if x == 4: - print x, y + print(x, y) #: E203:12 x, y = y , x # Okay if x == 4: - print x, y + print(x, y) x, y = y, x a[b1, :] == a[b1, ...] a[b1, :1] == 3 diff --git a/test/normalizer_issue_files/E27.py b/test/normalizer_issue_files/E27.py index c6221f5..0c328e3 100644 --- a/test/normalizer_issue_files/E27.py +++ b/test/normalizer_issue_files/E27.py @@ -22,6 +22,7 @@ True and False if 1: pass # Syntax Error, no indentation +#: E901+1 if 1: pass #: E223:8 diff --git a/test/test_python_errors.py b/test/test_python_errors.py index 50406f2..7e3449a 100644 --- a/test/test_python_errors.py +++ b/test/test_python_errors.py @@ -27,7 +27,7 @@ def _get_error_list(code, version=None): ('?', [(1, 0)]), ('??', [(1, 0)]), ('? ?', [(1, 0)]), - ('?\n?', [(1, 0)]), + ('?\n?', [(1, 0), (2, 0)]), ('? * ?', [(1, 0)]), ('1 + * * 2', [(1, 4)]), ('?\n1\n?', [(1, 0), (3, 0)]),