From 0fecb0a7800995a7ae28b6a7e2ee59624ed9a249 Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Wed, 22 May 2013 22:33:50 +0200 Subject: [PATCH 01/10] Add sith.py, a script for random smoke test --- sith.py | 168 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 168 insertions(+) create mode 100755 sith.py diff --git a/sith.py b/sith.py new file mode 100755 index 00000000..c51552e0 --- /dev/null +++ b/sith.py @@ -0,0 +1,168 @@ +#!/usr/bin/env python + +""" +Sith attacks (and helps debugging) Jedi. +""" + +import json +import os +import random +import sys +import traceback + +import jedi + + +class SourceCode(object): + + def __init__(self, path): + self.path = path + with open(path) as f: + self.source = f.read() + self.lines = self.source.splitlines() + self.maxline = len(self.lines) + + def choose_script_args(self): + line = random.randint(1, self.maxline) + column = random.randint(0, len(self.lines[line - 1])) + return (self.source, line, column, self.path) + + +class SourceFinder(object): + + def __init__(self, rootpath): + self.rootpath = rootpath + self.files = list(self.search_files()) + + def search_files(self): + for root, dirnames, filenames in os.walk(self.rootpath): + for name in filenames: + if name.endswith('.py'): + yield os.path.join(root, name) + + def choose_source(self): + # FIXME: try same file for several times + return SourceCode(random.choice(self.files)) + + +class BaseAttacker(object): + + def __init__(self): + self.record = {'data': []} + + def attack(self, operation, *args): + script = jedi.Script(*args) + op = getattr(script, operation) + op() + + def add_record(self, exc_info, operation, args): + (_type, value, tb) = exc_info + self.record['data'].append({ + 'traceback': traceback.format_tb(tb), + 'error': repr(value), + 'operation': operation, + 'args': args, + }) + + def get_record(self, recid): + return self.record['data'][recid] + + def save_record(self, path): + with open(path, 'w') as f: + json.dump(self.record, f) + + def load_record(self, path): + with open(path) as f: + self.record = json.load(f) + return self.record + + def add_arguments(self, parser): + parser.set_defaults(func=self.do_run) + + +class RandomAtaccker(BaseAttacker): + + operations = [ + 'completions', 'goto_assignments', 'goto_definitions', 'usages', + 'call_signatures'] + + def choose_operation(self): + return random.choice(self.operations) + + def generate_attacks(self, maxtries, finder): + for _ in range(maxtries): + src = finder.choose_source() + operation = self.choose_operation() + yield (operation, src.choose_script_args()) + + def do_run(self, record, rootpath, maxtries): + finder = SourceFinder(rootpath) + for (operation, args) in self.generate_attacks(maxtries, finder): + try: + self.attack(operation, *args) + except Exception: + self.add_record(sys.exc_info(), operation, args) + break + self.save_record(record) + + def add_arguments(self, parser): + super(RandomAtaccker, self).add_arguments(parser) + parser.add_argument( + '--maxtries', default=10000, type=int) + parser.add_argument( + 'rootpath', default='.', nargs='?', + help='root directory to look for Python files.') + + +class RedoAttacker(BaseAttacker): + + def do_run(self, record, recid): + self.load_record(record) + data = self.get_record(recid) + self.attack(data['operation'], *data['args']) + + def add_arguments(self, parser): + super(RedoAttacker, self).add_arguments(parser) + parser = parser.add_argument( + 'recid', default=0, type=int) + + +class AttackApp(object): + + def __init__(self): + self.parsers = [] + self.attackers = [] + + def run(self, args=None): + parser = self.get_parser() + self.do_run(**vars(parser.parse_args(args))) + + def do_run(self, func, **kwds): + func(**kwds) + + def add_parser(self, attacker_class, *args, **kwds): + parser = self.subparsers.add_parser(*args, **kwds) + attacker = attacker_class() + attacker.add_arguments(parser) + + # Not required, just fore debugging: + self.parsers.append(parser) + self.attackers.append(attacker) + + def get_parser(self): + import argparse + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + '--record', default='record.json', + help='Exceptions are recorded in here.') + + self.subparsers = parser.add_subparsers() + self.add_parser(RandomAtaccker, 'random', help='Random attack') + self.add_parser(RedoAttacker, 'redo', help='Redo recorded attack') + + return parser + + +if __name__ == '__main__': + app = AttackApp() + app.run() From 311025258e872d24f412b9b5b4e9da4d82671c0a Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Wed, 22 May 2013 22:51:34 +0200 Subject: [PATCH 02/10] Print exception --- sith.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/sith.py b/sith.py index c51552e0..4d359fcb 100755 --- a/sith.py +++ b/sith.py @@ -4,6 +4,7 @@ Sith attacks (and helps debugging) Jedi. """ +from __future__ import print_function import json import os import random @@ -80,7 +81,20 @@ class BaseAttacker(object): parser.set_defaults(func=self.do_run) -class RandomAtaccker(BaseAttacker): +class MixinPrinter(object): + + def print_record(self, recid=-1): + data = self.get_record(recid) + print(*data['traceback'], end='') + print(""" +{error} is raised by running Script(...).{operation}() with +line : {args[1]} +column: {args[2]} +path : {args[3]} +""".format(**data)) + + +class RandomAtaccker(MixinPrinter, BaseAttacker): operations = [ 'completions', 'goto_assignments', 'goto_definitions', 'usages', @@ -102,6 +116,7 @@ class RandomAtaccker(BaseAttacker): self.attack(operation, *args) except Exception: self.add_record(sys.exc_info(), operation, args) + self.print_record() break self.save_record(record) From 1e209aed37bb4cab8899b88d3e716a3a7161f946 Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Wed, 22 May 2013 22:56:25 +0200 Subject: [PATCH 03/10] Add 'show' command --- sith.py | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/sith.py b/sith.py index 4d359fcb..e8259374 100755 --- a/sith.py +++ b/sith.py @@ -94,6 +94,17 @@ path : {args[3]} """.format(**data)) +class MixinLoader(object): + + def add_arguments(self, parser): + super(MixinLoader, self).add_arguments(parser) + parser = parser.add_argument( + 'recid', default=0, nargs='?', type=int) + + def do_run(self, record, recid): + self.load_record(record) + + class RandomAtaccker(MixinPrinter, BaseAttacker): operations = [ @@ -129,17 +140,19 @@ class RandomAtaccker(MixinPrinter, BaseAttacker): help='root directory to look for Python files.') -class RedoAttacker(BaseAttacker): +class RedoAttacker(MixinLoader, BaseAttacker): def do_run(self, record, recid): - self.load_record(record) + super(RedoAttacker, self).do_run(record, recid) data = self.get_record(recid) self.attack(data['operation'], *data['args']) - def add_arguments(self, parser): - super(RedoAttacker, self).add_arguments(parser) - parser = parser.add_argument( - 'recid', default=0, type=int) + +class ShowRecord(MixinLoader, MixinPrinter, BaseAttacker): + + def do_run(self, record, recid): + super(ShowRecord, self).do_run(record, recid) + self.print_record() class AttackApp(object): @@ -174,6 +187,7 @@ class AttackApp(object): self.subparsers = parser.add_subparsers() self.add_parser(RandomAtaccker, 'random', help='Random attack') self.add_parser(RedoAttacker, 'redo', help='Redo recorded attack') + self.add_parser(ShowRecord, 'show', help='Show record') return parser From 6d026ec1af02712459aa013e74eee920cb6835cc Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Wed, 22 May 2013 23:11:50 +0200 Subject: [PATCH 04/10] Add --(i)pdb option --- sith.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/sith.py b/sith.py index e8259374..6ff37ca4 100755 --- a/sith.py +++ b/sith.py @@ -128,7 +128,7 @@ class RandomAtaccker(MixinPrinter, BaseAttacker): except Exception: self.add_record(sys.exc_info(), operation, args) self.print_record() - break + raise self.save_record(record) def add_arguments(self, parser): @@ -165,8 +165,17 @@ class AttackApp(object): parser = self.get_parser() self.do_run(**vars(parser.parse_args(args))) - def do_run(self, func, **kwds): - func(**kwds) + def do_run(self, func, debugger, **kwds): + try: + func(**kwds) + except: + exc_info = sys.exc_info() + if debugger == 'pdb': + import pdb + pdb.post_mortem(exc_info[2]) + elif debugger == 'ipdb': + import ipdb + ipdb.post_mortem(exc_info[2]) def add_parser(self, attacker_class, *args, **kwds): parser = self.subparsers.add_parser(*args, **kwds) @@ -183,6 +192,10 @@ class AttackApp(object): parser.add_argument( '--record', default='record.json', help='Exceptions are recorded in here.') + parser.add_argument( + '--pdb', dest='debugger', const='pdb', action='store_const') + parser.add_argument( + '--ipdb', dest='debugger', const='ipdb', action='store_const') self.subparsers = parser.add_subparsers() self.add_parser(RandomAtaccker, 'random', help='Random attack') From 2dee71ff4bbd82106ab27512d834ac0a5a352018 Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Wed, 22 May 2013 23:14:32 +0200 Subject: [PATCH 05/10] Ignore jedi.NotFoundError --- sith.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sith.py b/sith.py index 6ff37ca4..e009bd32 100755 --- a/sith.py +++ b/sith.py @@ -125,6 +125,8 @@ class RandomAtaccker(MixinPrinter, BaseAttacker): for (operation, args) in self.generate_attacks(maxtries, finder): try: self.attack(operation, *args) + except jedi.NotFoundError: + pass except Exception: self.add_record(sys.exc_info(), operation, args) self.print_record() From 8bf5f9d539e74284f4e5957e47e2df0d1cf5423c Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Wed, 22 May 2013 23:31:42 +0200 Subject: [PATCH 06/10] Report attacking by "." --- sith.py | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/sith.py b/sith.py index e009bd32..6af7cabf 100755 --- a/sith.py +++ b/sith.py @@ -11,6 +11,11 @@ import random import sys import traceback +try: + from itertools import izip as zip +except ImportError: + pass + import jedi @@ -105,6 +110,31 @@ class MixinLoader(object): self.load_record(record) +class AttackReporter(object): + + def __init__(self): + self.tries = 0 + self.errors = 0 + + def __iter__(self): + return self + + def __next__(self): + self.tries += 1 + sys.stderr.write('.') + sys.stderr.flush() + return self.tries + + next = __next__ + + def error(self): + self.errors += 1 + sys.stderr.write('\n') + sys.stderr.flush() + print('{0}th error is encountered after {1} tries.' + .format(self.errors, self.tries)) + + class RandomAtaccker(MixinPrinter, BaseAttacker): operations = [ @@ -122,13 +152,16 @@ class RandomAtaccker(MixinPrinter, BaseAttacker): def do_run(self, record, rootpath, maxtries): finder = SourceFinder(rootpath) + reporter = AttackReporter() for (operation, args) in self.generate_attacks(maxtries, finder): + reporter.next() try: self.attack(operation, *args) except jedi.NotFoundError: pass except Exception: self.add_record(sys.exc_info(), operation, args) + reporter.error() self.print_record() raise self.save_record(record) From cee0d4cf2fc5805bea8cc872a96414454dd8db3a Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Wed, 22 May 2013 23:49:45 +0200 Subject: [PATCH 07/10] Generate better help / add more help --- sith.py | 60 +++++++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 52 insertions(+), 8 deletions(-) diff --git a/sith.py b/sith.py index 6af7cabf..971f07b8 100755 --- a/sith.py +++ b/sith.py @@ -2,6 +2,21 @@ """ Sith attacks (and helps debugging) Jedi. + +Randomly search Python files and run Jedi on it. Exception and used +arguments are recorded to ``./record.josn`` (specified by --record):: + + %(prog)s random + +Redo recorded exception:: + + %(prog)s redo + +Fallback to pdb when error is raised:: + + %(prog)s --pdb random + %(prog)s --pdb redo + """ from __future__ import print_function @@ -85,6 +100,12 @@ class BaseAttacker(object): def add_arguments(self, parser): parser.set_defaults(func=self.do_run) + def get_help(self): + for line in self.__doc__.splitlines(): + line = line.strip() + if line: + return line + class MixinPrinter(object): @@ -104,7 +125,10 @@ class MixinLoader(object): def add_arguments(self, parser): super(MixinLoader, self).add_arguments(parser) parser = parser.add_argument( - 'recid', default=0, nargs='?', type=int) + 'recid', default=0, nargs='?', type=int, help=""" + This option currently has no effect as random attack record + only one error. + """) def do_run(self, record, recid): self.load_record(record) @@ -137,6 +161,10 @@ class AttackReporter(object): class RandomAtaccker(MixinPrinter, BaseAttacker): + """ + Randomly run Script().() against files under . + """ + operations = [ 'completions', 'goto_assignments', 'goto_definitions', 'usages', 'call_signatures'] @@ -177,6 +205,10 @@ class RandomAtaccker(MixinPrinter, BaseAttacker): class RedoAttacker(MixinLoader, BaseAttacker): + """ + Redo recorded attack. + """ + def do_run(self, record, recid): super(RedoAttacker, self).do_run(record, recid) data = self.get_record(recid) @@ -185,6 +217,10 @@ class RedoAttacker(MixinLoader, BaseAttacker): class ShowRecord(MixinLoader, MixinPrinter, BaseAttacker): + """ + Show recorded errors. + """ + def do_run(self, record, recid): super(ShowRecord, self).do_run(record, recid) self.print_record() @@ -213,8 +249,12 @@ class AttackApp(object): ipdb.post_mortem(exc_info[2]) def add_parser(self, attacker_class, *args, **kwds): - parser = self.subparsers.add_parser(*args, **kwds) attacker = attacker_class() + parser = self.subparsers.add_parser( + *args, + help=attacker.get_help(), + description=attacker.__doc__, + **kwds) attacker.add_arguments(parser) # Not required, just fore debugging: @@ -223,19 +263,23 @@ class AttackApp(object): def get_parser(self): import argparse - parser = argparse.ArgumentParser(description=__doc__) + parser = argparse.ArgumentParser( + formatter_class=argparse.RawDescriptionHelpFormatter, + description=__doc__) parser.add_argument( '--record', default='record.json', help='Exceptions are recorded in here.') parser.add_argument( - '--pdb', dest='debugger', const='pdb', action='store_const') + '--pdb', dest='debugger', const='pdb', action='store_const', + help="Launch pdb when error is raised.") parser.add_argument( - '--ipdb', dest='debugger', const='ipdb', action='store_const') + '--ipdb', dest='debugger', const='ipdb', action='store_const', + help="Launch ipdb when error is raised.") self.subparsers = parser.add_subparsers() - self.add_parser(RandomAtaccker, 'random', help='Random attack') - self.add_parser(RedoAttacker, 'redo', help='Redo recorded attack') - self.add_parser(ShowRecord, 'show', help='Show record') + self.add_parser(RandomAtaccker, 'random') + self.add_parser(RedoAttacker, 'redo') + self.add_parser(ShowRecord, 'show') return parser From fd0ec772fbd8588c4fbdc8d4022b39f98e0eda17 Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Thu, 23 May 2013 00:20:11 +0200 Subject: [PATCH 08/10] Fix: record was not saved --- sith.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sith.py b/sith.py index 971f07b8..c40ff27a 100755 --- a/sith.py +++ b/sith.py @@ -192,7 +192,8 @@ class RandomAtaccker(MixinPrinter, BaseAttacker): reporter.error() self.print_record() raise - self.save_record(record) + finally: + self.save_record(record) def add_arguments(self, parser): super(RandomAtaccker, self).add_arguments(parser) From d246192df029a17491c48a17a8594bf106757359 Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Thu, 23 May 2013 00:49:20 +0200 Subject: [PATCH 09/10] Add short options --- sith.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sith.py b/sith.py index c40ff27a..df53c623 100755 --- a/sith.py +++ b/sith.py @@ -198,7 +198,7 @@ class RandomAtaccker(MixinPrinter, BaseAttacker): def add_arguments(self, parser): super(RandomAtaccker, self).add_arguments(parser) parser.add_argument( - '--maxtries', default=10000, type=int) + '--maxtries', '-l', default=10000, type=int) parser.add_argument( 'rootpath', default='.', nargs='?', help='root directory to look for Python files.') @@ -268,7 +268,7 @@ class AttackApp(object): formatter_class=argparse.RawDescriptionHelpFormatter, description=__doc__) parser.add_argument( - '--record', default='record.json', + '--record', '-R', default='record.json', help='Exceptions are recorded in here.') parser.add_argument( '--pdb', dest='debugger', const='pdb', action='store_const', From 286279f14a126272fc356cb5ca463292255cf274 Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Thu, 23 May 2013 00:53:45 +0200 Subject: [PATCH 10/10] Show default value for --record --- sith.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sith.py b/sith.py index df53c623..dbde8a84 100755 --- a/sith.py +++ b/sith.py @@ -269,7 +269,7 @@ class AttackApp(object): description=__doc__) parser.add_argument( '--record', '-R', default='record.json', - help='Exceptions are recorded in here.') + help='Exceptions are recorded in here (default: %(default)s).') parser.add_argument( '--pdb', dest='debugger', const='pdb', action='store_const', help="Launch pdb when error is raised.")