VimPlug/jedi-fork
1
0
Fork 0
forked from VimPlug/jedi
Code Pull Requests Activity

pep correct python code

Browse Source
This commit is contained in:
David Halter
2012-02-21 00:21:00 +01:00
parent 49b7e778ce
commit 7597ed05a0
2 changed files with 157 additions and 106 deletions
Download Patch File Download Diff File Expand all files Collapse all files

115
pycomplete.py
View File

@@ -4,10 +4,10 @@ import pyclbr
import cStringIO import cStringIO
import sys import sys
import types import types
from token import ENDMARKER , NT_OFFSET , NUMBER , STRING , NEWLINE , INDENT , DEDENT , LPAR , RPAR , LSQB , RSQB , COLON , COMMA , SEMI , PLUS , MINUS , STAR , SLASH , VBAR , AMPER , LESS , GREATER , EQUAL , DOT , PERCENT , BACKQUOTE , LBRACE , RBRACE , EQEQUAL , NOTEQUAL , LESSEQUAL , GREATEREQUAL , TILDE , CIRCUMFLEX , LEFTSHIFT , RIGHTSHIFT , DOUBLESTAR , PLUSEQUAL , MINEQUAL , STAREQUAL , SLASHEQUAL , PERCENTEQUAL , AMPEREQUAL , VBAREQUAL , CIRCUMFLEXEQUAL , LEFTSHIFTEQUAL , RIGHTSHIFTEQUAL , DOUBLESTAREQUAL , DOUBLESLASH , DOUBLESLASHEQUAL , AT , NAME , ERRORTOKEN , N_TOKENS , OP
from pyfuzzyparser import PyFuzzyParser, _sanitize from pyfuzzyparser import PyFuzzyParser, _sanitize
def complete(file_name, line, colon): def complete(file_name, line, colon):
options = [] options = []
print file_name, line, colon print file_name, line, colon
@@ -24,55 +24,62 @@ def complete(file_name, line, colon):
pass pass
return options return options
print 1 print 1
if __name__ == '__main__': if __name__ == '__main__':
#print complete('test.py', 50, 3) #print complete('test.py', 50, 3)
#print complete('test.py', 51, 10) #print complete('test.py', 51, 10)
pass pass
class Completer(object): class Completer(object):
def __init__(self): def __init__(self):
self.compldict = {} self.compldict = {}
self.parser = PyFuzzyParser() self.parser = PyFuzzyParser()
def evalsource(self,text,line=0): def evalsource(self, text, line=0):
sc = self.parser.parse(text,line) sc = self.parser.parse(text, line)
self.sc = sc # TODO rm self.sc = sc # TODO rm
src = sc.get_code() src = sc.get_code()
dbg("source: %s" % src) dbg("source: %s" % src)
#try: exec(src) in self.compldict #try: exec(src) in self.compldict
#except: dbg("parser: %s, %s" % (sys.exc_info()[0],sys.exc_info()[1])) #except: dbg("parser: %s, %s" % (sys.exc_info()[0], sys.exc_info()[1]))
#for l in sc.locals: #for l in sc.locals:
# dbg("local: %s" % l) # dbg("local: %s" % l)
# try: exec(l) in self.compldict # try: exec(l) in self.compldict
# except: dbg("locals: %s, %s [%s]" % (sys.exc_info()[0],sys.exc_info()[1],l)) # except: dbg("locals: %s, %s [%s]" % (sys.exc_info()[0], sys.exc_info()[1], l))
def _cleanstr(self,doc): def _cleanstr(self, doc):
return doc.replace('"',' ').replace("'",' ') return doc.replace('"', ' ').replace("'", ' ')
def get_arguments(self,func_obj): def get_arguments(self, func_obj):
def _ctor(obj): def _ctor(obj):
try: return class_ob.__init__.im_func try:
return class_ob.__init__.im_func
except AttributeError: except AttributeError:
for base in class_ob.__bases__: for base in class_ob.__bases__:
rc = _find_constructor(base) rc = _find_constructor(base)
if rc is not None: return rc if rc is not None:
return rc
return None return None
arg_offset = 1 arg_offset = 1
if type(func_obj) == types.ClassType: func_obj = _ctor(func_obj) if type(func_obj) == types.ClassType:
elif type(func_obj) == types.MethodType: func_obj = func_obj.im_func func_obj = _ctor(func_obj)
else: arg_offset = 0 elif type(func_obj) == types.MethodType:
func_obj = func_obj.im_func
else:
arg_offset = 0
arg_text='' arg_text = ''
if type(func_obj) in [types.FunctionType, types.LambdaType]: if type(func_obj) in [types.FunctionType, types.LambdaType]:
try: try:
cd = func_obj.func_code cd = func_obj.func_code
real_args = cd.co_varnames[arg_offset:cd.co_argcount] real_args = cd.co_varnames[arg_offset:cd.co_argcount]
defaults = func_obj.func_defaults or '' defaults = func_obj.func_defaults or ''
defaults = map(lambda name: "=%s" % name, defaults) defaults = map(lambda name: "=%s" % name, defaults)
defaults = [""] * (len(real_args)-len(defaults)) + defaults defaults = [""] * (len(real_args) - len(defaults)) + defaults
items = map(lambda a,d: a+d, real_args, defaults) items = map(lambda a, d: a + d, real_args, defaults)
if func_obj.func_code.co_flags & 0x4: if func_obj.func_code.co_flags & 0x4:
items.append("...") items.append("...")
if func_obj.func_code.co_flags & 0x8: if func_obj.func_code.co_flags & 0x8:
@@ -80,7 +87,7 @@ class Completer(object):
arg_text = (','.join(items)) + ')' arg_text = (','.join(items)) + ')'
except: except:
dbg("arg completion: %s: %s" % (sys.exc_info()[0],sys.exc_info()[1])) dbg("arg completion: %s: %s" % (sys.exc_info()[0], sys.exc_info()[1]))
pass pass
if len(arg_text) == 0: if len(arg_text) == 0:
# The doc string sometimes contains the function signature # The doc string sometimes contains the function signature
@@ -95,15 +102,18 @@ class Completer(object):
lidx = sigline.find('(') lidx = sigline.find('(')
ridx = sigline.find(')') ridx = sigline.find(')')
if lidx > 0 and ridx > 0: if lidx > 0 and ridx > 0:
arg_text = sigline[lidx+1:ridx] + ')' arg_text = sigline[lidx + 1:ridx] + ')'
if len(arg_text) == 0: arg_text = ')' if len(arg_text) == 0:
arg_text = ')'
return arg_text return arg_text
def get_completions(self,context,match): def get_completions(self, context, match):
dbg("get_completions('%s','%s')" % (context,match)) dbg("get_completions('%s','%s')" % (context, match))
stmt = '' stmt = ''
if context: stmt += str(context) if context:
if match: stmt += str(match) stmt += str(context)
if match:
stmt += str(match)
try: try:
result = None result = None
all = {} all = {}
@@ -111,14 +121,15 @@ class Completer(object):
if len(stmt) > 0 and stmt[-1] == '(': if len(stmt) > 0 and stmt[-1] == '(':
result = eval(_sanitize(stmt[:-1]), self.compldict) result = eval(_sanitize(stmt[:-1]), self.compldict)
doc = result.__doc__ doc = result.__doc__
if doc is None: doc = '' if doc is None:
doc = ''
args = self.get_arguments(result) args = self.get_arguments(result)
return [{'word':self._cleanstr(args),'info':self._cleanstr(doc)}] return [{'word': self._cleanstr(args), 'info': self._cleanstr(doc)}]
elif ridx == -1: elif ridx == -1:
match = stmt match = stmt
all = self.compldict all = self.compldict
else: else:
match = stmt[ridx+1:] match = stmt[ridx + 1:]
stmt = _sanitize(stmt[:ridx]) stmt = _sanitize(stmt[:ridx])
result = eval(stmt, self.compldict) result = eval(stmt, self.compldict)
all = dir(result) all = dir(result)
@@ -126,23 +137,32 @@ class Completer(object):
dbg("completing: stmt:%s" % stmt) dbg("completing: stmt:%s" % stmt)
completions = [] completions = []
try: maindoc = result.__doc__ try:
except: maindoc = ' ' maindoc = result.__doc__
if maindoc is None: maindoc = ' ' except:
maindoc = ' '
if maindoc is None:
maindoc = ' '
for m in all: for m in all:
if m == "_PyCmplNoType": continue #this is internal if m == "_PyCmplNoType":
continue # this is internal
try: try:
dbg('possible completion: %s' % m) dbg('possible completion: %s' % m)
if m.find(match) == 0: if m.find(match) == 0:
if result is None: inst = all[m] if result is None:
else: inst = getattr(result,m) inst = all[m]
try: doc = inst.__doc__ else:
except: doc = maindoc inst = getattr(result, m)
try:
doc = inst.__doc__
except:
doc = maindoc
typestr = str(inst) typestr = str(inst)
if doc is None or doc == '': doc = maindoc if doc is None or doc == '':
doc = maindoc
wrd = m[len(match):] wrd = m[len(match):]
c = {'word':wrd, 'abbr':m, 'info':self._cleanstr(doc)} c = {'word': wrd, 'abbr': m, 'info': self._cleanstr(doc)}
if "function" in typestr: if "function" in typestr:
c['word'] += '(' c['word'] += '('
c['abbr'] += '(' + self._cleanstr(self.get_arguments(inst)) c['abbr'] += '(' + self._cleanstr(self.get_arguments(inst))
@@ -157,23 +177,26 @@ class Completer(object):
completions.append(c) completions.append(c)
except: except:
i = sys.exc_info() i = sys.exc_info()
dbg("inner completion: %s,%s [stmt='%s']" % (i[0],i[1],stmt)) dbg("inner completion: %s, %s [stmt='%s']" % (i[0], i[1], stmt))
return completions return completions
except: except:
i = sys.exc_info() i = sys.exc_info()
dbg("completion: %s,%s [stmt='%s']" % (i[0],i[1],stmt)) dbg("completion: %s, %s [stmt='%s']" % (i[0], i[1], stmt))
return [] return []
debugstmts=[] debugstmts = []
def dbg(s): debugstmts.append(s) def dbg(s):
def showdbg(): debugstmts.append(s)
for d in debugstmts: print "DBG: %s " % d
def showdbg():
for d in debugstmts:
print "DBG: %s " % d
text = cStringIO.StringIO(open('test.py').read()) text = cStringIO.StringIO(open('test.py').read())
cmpl = Completer() cmpl = Completer()
cmpl.evalsource(text,51) cmpl.evalsource(text, 51)
print cmpl.sc.get_code() print cmpl.sc.get_code()
all = cmpl.get_completions("cdef.", '') all = cmpl.get_completions("cdef.", '')

148
pyfuzzyparser.py
View File

@@ -5,13 +5,15 @@ import sys
import tokenize import tokenize
import cStringIO import cStringIO
def indent_block(text, indention=" "): def indent_block(text, indention=" "):
""" This function indents a text block with a default of four spaces """ """ This function indents a text block with a default of four spaces """
lines = text.split('\n') lines = text.split('\n')
return '\n'.join(map(lambda s: indention+s, lines)) return '\n'.join(map(lambda s: indention + s, lines))
class Scope(object): class Scope(object):
def __init__(self,name,indent,docstr=''): def __init__(self, name, indent, docstr=''):
self.subscopes = [] self.subscopes = []
self.locals = [] self.locals = []
self.imports = [] self.imports = []
@@ -20,29 +22,32 @@ class Scope(object):
self.name = name self.name = name
self.indent = indent self.indent = indent
def add_scope(self,sub): def add_scope(self, sub):
#print 'push scope: [%s@%s]' % (sub.name,sub.indent) #print 'push scope: [%s@%s]' % (sub.name, sub.indent)
sub.parent = self sub.parent = self
self.subscopes.append(sub) self.subscopes.append(sub)
return sub return sub
def doc(self,str): def doc(self, str):
""" Clean up a docstring """ """ Clean up a docstring """
d = str.replace('\n',' ') d = str.replace('\n', ' ')
d = d.replace('\t',' ') d = d.replace('\t', ' ')
while d.find(' ') > -1: d = d.replace(' ',' ') while d.find(' ') > -1:
while d[0] in '"\'\t ': d = d[1:] d = d.replace(' ', ' ')
while d[-1] in '"\'\t ': d = d[:-1] while d[0] in '"\'\t ':
dbg("Scope(%s)::docstr = %s" % (self,d)) d = d[1:]
while d[-1] in '"\'\t ':
d = d[:-1]
dbg("Scope(%s)::docstr = %s" % (self, d))
self.docstr = d self.docstr = d
def add_local(self,loc): def add_local(self, loc):
self.locals.append(loc) self.locals.append(loc)
def add_import(self, imp): def add_import(self, imp):
self.imports.append(imp) self.imports.append(imp)
def _checkexisting(self,test): def _checkexisting(self, test):
"Convienance function... keep out duplicates" "Convienance function... keep out duplicates"
if test.find('=') > -1: if test.find('=') > -1:
var = test.split('=')[0].strip() var = test.split('=')[0].strip()
@@ -52,25 +57,26 @@ class Scope(object):
def get_code(self, first_indent=False, indention=" "): def get_code(self, first_indent=False, indention=" "):
str = "" str = ""
#str += 'class _PyCmplNoType:\n def __getattr__(self,name):\n return None\n' if len(self.docstr) > 0:
if len(self.docstr) > 0: str += '"""'+self.docstr+'"""\n' str += '"""' + self.docstr + '"""\n'
for i in self.imports: for i in self.imports:
str += i.get_code() + '\n' str += i.get_code() + '\n'
#str += 'class _PyCmplNoType:\n def __getattr__(self,name):\n return None\n'
for sub in self.subscopes: for sub in self.subscopes:
str += sub.get_code(first_indent=True, indention=indention) str += sub.get_code(first_indent=True, indention=indention)
for l in self.locals: for l in self.locals:
str += l+'\n' str += l + '\n'
if first_indent: str = indent_block(str, indention = indention) if first_indent:
str = indent_block(str, indention=indention)
return "_%s_%s" % (self.indent, str) return "_%s_%s" % (self.indent, str)
def is_empty(self): def is_empty(self):
""" """
this function returns true if there are no subscopes, imports, locals. this function returns true if there are no subscopes, imports, locals.
""" """
return not (self.locals, self.imports, self.subscopes) return not (self.locals, self.imports, self.subscopes)
class Class(Scope): class Class(Scope):
def __init__(self, name, supers, indent, docstr=''): def __init__(self, name, supers, indent, docstr=''):
super(Class, self).__init__(name, indent, docstr) super(Class, self).__init__(name, indent, docstr)
@@ -78,7 +84,8 @@ class Class(Scope):
def get_code(self, first_indent=False, indention=" "): def get_code(self, first_indent=False, indention=" "):
str = 'class %s' % (self.name) str = 'class %s' % (self.name)
if len(self.supers) > 0: str += '(%s)' % ','.join(self.supers) if len(self.supers) > 0:
str += '(%s)' % ','.join(self.supers)
str += ':\n' str += ':\n'
str += super(Class, self).get_code(True, indention) str += super(Class, self).get_code(True, indention)
if self.is_empty(): if self.is_empty():
@@ -88,18 +95,20 @@ class Class(Scope):
class Function(Scope): class Function(Scope):
def __init__(self, name, params, indent, docstr=''): def __init__(self, name, params, indent, docstr=''):
Scope.__init__(self,name,indent, docstr) Scope.__init__(self, name, indent, docstr)
self.params = params self.params = params
def get_code(self, first_indent=False, indention=" "): def get_code(self, first_indent=False, indention=" "):
str = "def %s(%s):\n" % (self.name,','.join(self.params)) str = "def %s(%s):\n" % (self.name, ','.join(self.params))
#if len(self.docstr) > 0: str += self.childindent()+'"""'+self.docstr+'"""\n' #if len(self.docstr) > 0:
# str += self.childindent()+'"""'+self.docstr+'"""\n'
str += super(Function, self).get_code(True, indention) str += super(Function, self).get_code(True, indention)
if self.is_empty(): if self.is_empty():
str += indent_block("pass\n", indention=indention) str += indent_block("pass\n", indention=indention)
print "func", self.locals print "func", self.locals
return str return str
class Import(object): class Import(object):
""" """
stores the imports of class files stores the imports of class files
@@ -128,29 +137,33 @@ class Import(object):
else: else:
return "test import " + ns_str return "test import " + ns_str
class PyFuzzyParser(object): class PyFuzzyParser(object):
""" """
This class is used to parse a Python file, it then divides them into a This class is used to parse a Python file, it then divides them into a
class structure of differnt scopes. class structure of differnt scopes.
""" """
def __init__(self): def __init__(self):
self.top = Scope('global',0) self.top = Scope('global', 0)
self.scope = self.top self.scope = self.top
def _parsedotname(self,pre=None): def _parsedotname(self, pre=None):
#returns (dottedname, nexttoken) #returns (dottedname, nexttoken)
name = [] name = []
if pre is None: if pre is None:
tokentype, tok, indent = self.next() tokentype, tok, indent = self.next()
if tokentype != tokenize.NAME and tok != '*': if tokentype != tokenize.NAME and tok != '*':
return ('', tok) return ('', tok)
else: tok = pre else:
tok = pre
name.append(tok) name.append(tok)
while True: while True:
tokentype, tok, indent = self.next() tokentype, tok, indent = self.next()
if tok != '.': break if tok != '.':
break
tokentype, tok, indent = self.next() tokentype, tok, indent = self.next()
if tokentype != tokenize.NAME: break if tokentype != tokenize.NAME:
break
name.append(tok) name.append(tok)
return (".".join(name), tok) return (".".join(name), tok)
@@ -158,13 +171,16 @@ class PyFuzzyParser(object):
imports = [] imports = []
while True: while True:
name, tok = self._parsedotname() name, tok = self._parsedotname()
if not name: break if not name:
break
name2 = '' name2 = ''
if tok == 'as': name2, tok = self._parsedotname() if tok == 'as':
name2, tok = self._parsedotname()
imports.append((name, name2)) imports.append((name, name2))
while tok != "," and "\n" not in tok: while tok != "," and "\n" not in tok:
tokentype, tok, indent = self.next() tokentype, tok, indent = self.next()
if tok != ",": break if tok != ",":
break
return imports return imports
def _parseparen(self): def _parseparen(self):
@@ -174,7 +190,8 @@ class PyFuzzyParser(object):
while True: while True:
tokentype, tok, indent = self.next() tokentype, tok, indent = self.next()
if tok in (')', ',') and level == 1: if tok in (')', ',') and level == 1:
if '=' not in name: name = name.replace(' ', '') if '=' not in name:
name = name.replace(' ', '')
names.append(name.strip()) names.append(name.strip())
name = '' name = ''
if tok == '(': if tok == '(':
@@ -182,8 +199,10 @@ class PyFuzzyParser(object):
name += "(" name += "("
elif tok == ')': elif tok == ')':
level -= 1 level -= 1
if level == 0: break if level == 0:
else: name += ")" break
else:
name += ")"
elif tok == ',' and level == 1: elif tok == ',' and level == 1:
pass pass
else: else:
@@ -192,31 +211,36 @@ class PyFuzzyParser(object):
def _parsefunction(self, indent): def _parsefunction(self, indent):
tokentype, fname, ind = self.next() tokentype, fname, ind = self.next()
if tokentype != tokenize.NAME: return None if tokentype != tokenize.NAME:
return None
tokentype, open, ind = self.next() tokentype, open, ind = self.next()
if open != '(': return None if open != '(':
params=self._parseparen() return None
params = self._parseparen()
tokentype, colon, ind = self.next() tokentype, colon, ind = self.next()
if colon != ':': return None if colon != ':':
return None
return Function(fname, params, indent) return Function(fname, params, indent)
def _parseclass(self, indent): def _parseclass(self, indent):
tokentype, cname, ind = self.next() tokentype, cname, ind = self.next()
if tokentype != tokenize.NAME: return None if tokentype != tokenize.NAME:
return None
super = [] super = []
tokentype, next, ind = self.next() tokentype, next, ind = self.next()
if next == '(': if next == '(':
super=self._parseparen() super = self._parseparen()
elif next != ':': return None elif next != ':':
return None
return Class(cname,super,indent) return Class(cname, super, indent)
def _parseassignment(self): def _parseassignment(self):
assign='' assign = ''
tokentype, tok, indent = self.next() tokentype, tok, indent = self.next()
if tokentype == tokenize.STRING or tok == 'str': if tokentype == tokenize.STRING or tok == 'str':
return '""' return '""'
@@ -233,41 +257,42 @@ class PyFuzzyParser(object):
elif tok == 'None': elif tok == 'None':
return '_PyCmplNoType()' return '_PyCmplNoType()'
elif tok == 'type': elif tok == 'type':
return 'type(_PyCmplNoType)' #only for method resolution return 'type(_PyCmplNoType)' # only for method resolution
else: else:
assign += tok assign += tok
level = 0 level = 0
while True: while True:
tokentype, tok, indent = self.next() tokentype, tok, indent = self.next()
if tok in ('(','{','['): if tok in ('(', '{', '['):
level += 1 level += 1
elif tok in (']','}',')'): elif tok in (']', '}', ')'):
level -= 1 level -= 1
if level == 0: break if level == 0:
break
elif level == 0: elif level == 0:
if tok in (';','\n'): break if tok in (';', '\n'):
break
assign += tok assign += tok
return "%s" % assign return "%s" % assign
def next(self): def next(self):
type, tok, (lineno, indent), end, self.parserline = self.gen.next() type, tok, (lineno, indent), end, self.parserline = self.gen.next()
if lineno == self.curline: if lineno == self.curline:
#print 'line found [%s] scope=%s' % (line.replace('\n',''),self.scope.name)
self.currentscope = self.scope self.currentscope = self.scope
return (type, tok, indent) return (type, tok, indent)
#p.parse(vim.current.buffer[:],vim.eval("line('.')")) #p.parse(vim.current.buffer[:], vim.eval("line('.')"))
def parse(self,text,curline=0): def parse(self, text, curline=0):
self.curline = int(curline) self.curline = int(curline)
buf = cStringIO.StringIO(''.join(text) + '\n') buf = cStringIO.StringIO(''.join(text) + '\n')
self.gen = tokenize.generate_tokens(buf.readline) self.gen = tokenize.generate_tokens(buf.readline)
self.currentscope = self.scope self.currentscope = self.scope
try: try:
freshscope=True freshscope = True
while True: while True:
tokentype, tok, indent = self.next() tokentype, tok, indent = self.next()
dbg( 'main: tok=[%s] indent=[%s]' % (tok,indent)) dbg('main: tok=[%s] indent=[%s]' % (tok, indent))
if tokentype == tokenize.DEDENT: if tokentype == tokenize.DEDENT:
self.scope = self.scope.parent self.scope = self.scope.parent
@@ -302,33 +327,36 @@ class PyFuzzyParser(object):
self.scope.add_import(Import(name, alias, mod)) self.scope.add_import(Import(name, alias, mod))
freshscope = False freshscope = False
elif tokentype == tokenize.STRING: elif tokentype == tokenize.STRING:
if freshscope: self.scope.doc(tok) if freshscope:
self.scope.doc(tok)
elif tokentype == tokenize.NAME: elif tokentype == tokenize.NAME:
name,tok = self._parsedotname(tok) name, tok = self._parsedotname(tok)
if tok == '=': if tok == '=':
stmt = self._parseassignment() stmt = self._parseassignment()
dbg("parseassignment: %s = %s" % (name, stmt)) dbg("parseassignment: %s = %s" % (name, stmt))
if stmt != None: if stmt != None:
self.scope.add_local("%s = %s" % (name,stmt)) self.scope.add_local("%s = %s" % (name, stmt))
freshscope = False freshscope = False
except StopIteration: #thrown on EOF except StopIteration: # thrown on EOF
pass pass
#except: #except:
# dbg("parse error: %s, %s @ %s" % # dbg("parse error: %s, %s @ %s" %
# (sys.exc_info()[0], sys.exc_info()[1], self.parserline)) # (sys.exc_info()[0], sys.exc_info()[1], self.parserline))
return self.top #self._adjustvisibility() return self.top # self._adjustvisibility()
def _sanitize(str): def _sanitize(str):
val = '' val = ''
level = 0 level = 0
for c in str: for c in str:
if c in ('(','{','['): if c in ('(', '{', '['):
level += 1 level += 1
elif c in (']','}',')'): elif c in (']', '}', ')'):
level -= 1 level -= 1
elif level == 0: elif level == 0:
val += c val += c
return val return val
def dbg(*args): def dbg(*args):
print args print args