VimPlug/jedi-fork
1
0
Fork 0
forked from VimPlug/jedi
Code Pull Requests Activity

refactorings

Browse Source
This commit is contained in:
David Halter
2012-02-21 00:01:59 +01:00
parent d720f79baa
commit 49b7e778ce
3 changed files with 91 additions and 154 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
pycomplete.py
View File

@@ -40,12 +40,12 @@ class Completer(object):
self.sc = sc # TODO rm self.sc = sc # TODO rm
src = sc.get_code() src = sc.get_code()
dbg("source: %s" % src) dbg("source: %s" % src)
try: exec(src) in self.compldict #try: exec(src) in self.compldict
except: dbg("parser: %s, %s" % (sys.exc_info()[0],sys.exc_info()[1])) #except: dbg("parser: %s, %s" % (sys.exc_info()[0],sys.exc_info()[1]))
for l in sc.locals: #for l in sc.locals:
dbg("local: %s" % l) # dbg("local: %s" % l)
try: exec(l) in self.compldict # try: exec(l) in self.compldict
except: dbg("locals: %s, %s [%s]" % (sys.exc_info()[0],sys.exc_info()[1],l)) # except: dbg("locals: %s, %s [%s]" % (sys.exc_info()[0],sys.exc_info()[1],l))
def _cleanstr(self,doc): def _cleanstr(self,doc):
return doc.replace('"',' ').replace("'",' ') return doc.replace('"',' ').replace("'",' ')
@@ -184,7 +184,7 @@ for c in all:
print '' print ''
showdbg() showdbg()
cmpl.parser.print_parser_code() print cmpl.parser.top.get_code()
import code import code
sh = code.InteractiveConsole(locals=locals()) sh = code.InteractiveConsole(locals=locals())

227
pyfuzzyparser.py
View File

@@ -4,10 +4,9 @@ TODO This is a parser
import sys import sys
import tokenize import tokenize
import cStringIO import cStringIO
from token import ENDMARKER , NT_OFFSET , NUMBER , STRING , NEWLINE , INDENT , DEDENT , LPAR , RPAR , LSQB , RSQB , COLON , COMMA , SEMI , PLUS , MINUS , STAR , SLASH , VBAR , AMPER , LESS , GREATER , EQUAL , DOT , PERCENT , BACKQUOTE , LBRACE , RBRACE , EQEQUAL , NOTEQUAL , LESSEQUAL , GREATEREQUAL , TILDE , CIRCUMFLEX , LEFTSHIFT , RIGHTSHIFT , DOUBLESTAR , PLUSEQUAL , MINEQUAL , STAREQUAL , SLASHEQUAL , PERCENTEQUAL , AMPEREQUAL , VBAREQUAL , CIRCUMFLEXEQUAL , LEFTSHIFTEQUAL , RIGHTSHIFTEQUAL , DOUBLESTAREQUAL , DOUBLESLASH , DOUBLESLASHEQUAL , AT , NAME , ERRORTOKEN , N_TOKENS , OP
def indent(text, indention=" "): def indent_block(text, indention=" "):
""" This function indents a text with a default of four spaces """ """ This function indents a text block with a default of four spaces """
lines = text.split('\n') lines = text.split('\n')
return '\n'.join(map(lambda s: indention+s, lines)) return '\n'.join(map(lambda s: indention+s, lines))
@@ -43,10 +42,6 @@ class Scope(object):
def add_import(self, imp): def add_import(self, imp):
self.imports.append(imp) self.imports.append(imp)
def copy_decl(self,indent=0):
""" Copy a scope's declaration only, at the specified indent level - not local variables """
return Scope(self.name,indent,self.docstr)
def _checkexisting(self,test): def _checkexisting(self,test):
"Convienance function... keep out duplicates" "Convienance function... keep out duplicates"
if test.find('=') > -1: if test.find('=') > -1:
@@ -67,43 +62,27 @@ class Scope(object):
for l in self.locals: for l in self.locals:
str += l+'\n' str += l+'\n'
if first_indent: str = indent(str, indention = indention) if first_indent: str = indent_block(str, indention = indention)
return str return "_%s_%s" % (self.indent, str)
def pop(self,indent): def is_empty(self):
#print 'pop scope: [%s] to [%s]' % (self.indent,indent) """
outer = self this function returns true if there are no subscopes, imports, locals.
while outer.parent != None and outer.indent >= indent: """
outer = outer.parent return not (self.locals, self.imports, self.subscopes)
return outer
def currentindent(self):
#print 'parse current indent: %s' % self.indent
return ' '*self.indent
def childindent(self):
#print 'parse child indent: [%s]' % (self.indent+1)
return ' '*(self.indent+1)
class Class(Scope): class Class(Scope):
def __init__(self, name, supers, indent, docstr=''): def __init__(self, name, supers, indent, docstr=''):
super(Class, self).__init__(name,indent, docstr) super(Class, self).__init__(name, indent, docstr)
self.supers = supers self.supers = supers
def copy_decl(self,indent=0):
c = Class(self.name,self.supers,indent, self.docstr)
for s in self.subscopes:
c.add_scope(s.copy_decl(indent+1))
return c
def get_code(self, first_indent=False, indention=" "): def get_code(self, first_indent=False, indention=" "):
str = 'class %s' % (self.name) str = 'class %s' % (self.name)
if len(self.supers) > 0: str += '(%s)' % ','.join(self.supers) if len(self.supers) > 0: str += '(%s)' % ','.join(self.supers)
str += ':\n' str += ':\n'
str += super(Class, self).get_code(True, indention) str += super(Class, self).get_code(True, indention)
#if len(self.docstr) > 0: str += self.childindent()+'"""'+self.docstr+'"""\n' if self.is_empty():
#if len(self.subscopes) > 0: str += indent_block("pass\n", indention=indention)
# for s in self.subscopes: str += s.get_code()
#else:
# str += '%spass\n' % self.childindent()
return str return str
@@ -111,14 +90,13 @@ class Function(Scope):
def __init__(self, name, params, indent, docstr=''): def __init__(self, name, params, indent, docstr=''):
Scope.__init__(self,name,indent, docstr) Scope.__init__(self,name,indent, docstr)
self.params = params self.params = params
def copy_decl(self,indent=0):
return Function(self.name,self.params,indent, self.docstr)
def get_code(self, first_indent=False, indention=" "): def get_code(self, first_indent=False, indention=" "):
str = "def %s(%s):\n" % (self.name,','.join(self.params)) str = "def %s(%s):\n" % (self.name,','.join(self.params))
#if len(self.docstr) > 0: str += self.childindent()+'"""'+self.docstr+'"""\n' #if len(self.docstr) > 0: str += self.childindent()+'"""'+self.docstr+'"""\n'
str += super(Function, self).get_code(True, indention) str += super(Function, self).get_code(True, indention)
if not len(self.subscopes) and not len(self.imports): if self.is_empty():
str += indent("pass\n", indention=indention) str += indent_block("pass\n", indention=indention)
print "func", self.locals print "func", self.locals
return str return str
@@ -152,7 +130,8 @@ class Import(object):
class PyFuzzyParser(object): class PyFuzzyParser(object):
""" """
This class is used to parse a Python file, it then divides them into This class is used to parse a Python file, it then divides them into a
class structure of differnt scopes.
""" """
def __init__(self): def __init__(self):
self.top = Scope('global',0) self.top = Scope('global',0)
@@ -162,30 +141,30 @@ class PyFuzzyParser(object):
#returns (dottedname, nexttoken) #returns (dottedname, nexttoken)
name = [] name = []
if pre is None: if pre is None:
tokentype, token, indent = self.next() tokentype, tok, indent = self.next()
if tokentype != NAME and token != '*': if tokentype != tokenize.NAME and tok != '*':
return ('', token) return ('', tok)
else: token = pre else: tok = pre
name.append(token) name.append(tok)
while True: while True:
tokentype, token, indent = self.next() tokentype, tok, indent = self.next()
if token != '.': break if tok != '.': break
tokentype, token, indent = self.next() tokentype, tok, indent = self.next()
if tokentype != NAME: break if tokentype != tokenize.NAME: break
name.append(token) name.append(tok)
return (".".join(name), token) return (".".join(name), tok)
def _parseimportlist(self): def _parseimportlist(self):
imports = [] imports = []
while True: while True:
name, token = self._parsedotname() name, tok = self._parsedotname()
if not name: break if not name: break
name2 = '' name2 = ''
if token == 'as': name2, token = self._parsedotname() if tok == 'as': name2, tok = self._parsedotname()
imports.append((name, name2)) imports.append((name, name2))
while token != "," and "\n" not in token: while tok != "," and "\n" not in tok:
tokentype, token, indent = self.next() tokentype, tok, indent = self.next()
if token != ",": break if tok != ",": break
return imports return imports
def _parseparen(self): def _parseparen(self):
@@ -193,28 +172,27 @@ class PyFuzzyParser(object):
names = [] names = []
level = 1 level = 1
while True: while True:
tokentype, token, indent = self.next() tokentype, tok, indent = self.next()
if token in (')', ',') and level == 1: if tok in (')', ',') and level == 1:
if '=' not in name: name = name.replace(' ', '') if '=' not in name: name = name.replace(' ', '')
names.append(name.strip()) names.append(name.strip())
name = '' name = ''
if token == '(': if tok == '(':
level += 1 level += 1
name += "(" name += "("
elif token == ')': elif tok == ')':
level -= 1 level -= 1
if level == 0: break if level == 0: break
else: name += ")" else: name += ")"
elif token == ',' and level == 1: elif tok == ',' and level == 1:
pass pass
else: else:
name += "%s " % str(token) name += "%s " % str(tok)
return names return names
def _parsefunction(self,indent): def _parsefunction(self, indent):
self.scope=self.scope.pop(indent)
tokentype, fname, ind = self.next() tokentype, fname, ind = self.next()
if tokentype != NAME: return None if tokentype != tokenize.NAME: return None
tokentype, open, ind = self.next() tokentype, open, ind = self.next()
if open != '(': return None if open != '(': return None
@@ -223,12 +201,11 @@ class PyFuzzyParser(object):
tokentype, colon, ind = self.next() tokentype, colon, ind = self.next()
if colon != ':': return None if colon != ':': return None
return Function(fname,params,indent) return Function(fname, params, indent)
def _parseclass(self,indent): def _parseclass(self, indent):
self.scope=self.scope.pop(indent)
tokentype, cname, ind = self.next() tokentype, cname, ind = self.next()
if tokentype != NAME: return None if tokentype != tokenize.NAME: return None
super = [] super = []
tokentype, next, ind = self.next() tokentype, next, ind = self.next()
@@ -240,83 +217,44 @@ class PyFuzzyParser(object):
def _parseassignment(self): def _parseassignment(self):
assign='' assign=''
tokentype, token, indent = self.next() tokentype, tok, indent = self.next()
if tokentype == tokenize.STRING or token == 'str': if tokentype == tokenize.STRING or tok == 'str':
return '""' return '""'
elif token == '(' or token == 'tuple': elif tok == '(' or tok == 'tuple':
return '()' return '()'
elif token == '[' or token == 'list': elif tok == '[' or tok == 'list':
return '[]' return '[]'
elif token == '{' or token == 'dict': elif tok == '{' or tok == 'dict':
return '{}' return '{}'
elif tokentype == tokenize.NUMBER: elif tokentype == tokenize.NUMBER:
return '0' return '0'
elif token == 'open' or token == 'file': elif tok == 'open' or tok == 'file':
return 'file' return 'file'
elif token == 'None': elif tok == 'None':
return '_PyCmplNoType()' return '_PyCmplNoType()'
elif token == 'type': elif tok == 'type':
return 'type(_PyCmplNoType)' #only for method resolution return 'type(_PyCmplNoType)' #only for method resolution
else: else:
assign += token assign += tok
level = 0 level = 0
while True: while True:
tokentype, token, indent = self.next() tokentype, tok, indent = self.next()
if token in ('(','{','['): if tok in ('(','{','['):
level += 1 level += 1
elif token in (']','}',')'): elif tok in (']','}',')'):
level -= 1 level -= 1
if level == 0: break if level == 0: break
elif level == 0: elif level == 0:
if token in (';','\n'): break if tok in (';','\n'): break
assign += token assign += tok
return "%s" % assign return "%s" % assign
def next(self): def next(self):
type, token, (lineno, indent), end, self.parserline = self.gen.next() type, tok, (lineno, indent), end, self.parserline = self.gen.next()
if lineno == self.curline: if lineno == self.curline:
#print 'line found [%s] scope=%s' % (line.replace('\n',''),self.scope.name) #print 'line found [%s] scope=%s' % (line.replace('\n',''),self.scope.name)
self.currentscope = self.scope self.currentscope = self.scope
return (type, token, indent) return (type, tok, indent)
def _adjustvisibility(self):
newscope = Scope('result',0)
scp = self.currentscope
while scp != None:
if type(scp) == Function:
slice = 0
#Handle 'self' params
if scp.parent != None and type(scp.parent) == Class:
slice = 1
newscope.add_local('%s = %s' % (scp.params[0],scp.parent.name))
for p in scp.params[slice:]:
i = p.find('=')
if len(p) == 0: continue
pvar = ''
ptype = ''
if i == -1:
pvar = p
ptype = '_PyCmplNoType()'
else:
pvar = p[:i]
ptype = _sanitize(p[i+1:])
if pvar.startswith('**'):
pvar = pvar[2:]
ptype = '{}'
elif pvar.startswith('*'):
pvar = pvar[1:]
ptype = '[]'
newscope.add_local('%s = %s' % (pvar,ptype))
for s in scp.subscopes:
ns = s.copy_decl(0)
newscope.add_scope(ns)
for l in scp.locals: newscope.add_local(l)
scp = scp.parent
self.currentscope = newscope
return self.currentscope
#p.parse(vim.current.buffer[:],vim.eval("line('.')")) #p.parse(vim.current.buffer[:],vim.eval("line('.')"))
def parse(self,text,curline=0): def parse(self,text,curline=0):
@@ -328,12 +266,12 @@ class PyFuzzyParser(object):
try: try:
freshscope=True freshscope=True
while True: while True:
tokentype, token, indent = self.next() tokentype, tok, indent = self.next()
#dbg( 'main: token=[%s] indent=[%s]' % (token,indent)) dbg( 'main: tok=[%s] indent=[%s]' % (tok,indent))
if tokentype == DEDENT or token == "pass": if tokentype == tokenize.DEDENT:
self.scope = self.scope.pop(indent) self.scope = self.scope.parent
elif token == 'def': elif tok == 'def':
func = self._parsefunction(indent) func = self._parsefunction(indent)
if func is None: if func is None:
print "function: syntax error..." print "function: syntax error..."
@@ -341,7 +279,7 @@ class PyFuzzyParser(object):
dbg("new scope: function") dbg("new scope: function")
freshscope = True freshscope = True
self.scope = self.scope.add_scope(func) self.scope = self.scope.add_scope(func)
elif token == 'class': elif tok == 'class':
cls = self._parseclass(indent) cls = self._parseclass(indent)
if cls is None: if cls is None:
print "class: syntax error..." print "class: syntax error..."
@@ -349,26 +287,25 @@ class PyFuzzyParser(object):
freshscope = True freshscope = True
dbg("new scope: class") dbg("new scope: class")
self.scope = self.scope.add_scope(cls) self.scope = self.scope.add_scope(cls)
elif tok == 'import':
elif token == 'import':
imports = self._parseimportlist() imports = self._parseimportlist()
for mod, alias in imports: for mod, alias in imports:
self.scope.add_import(Import(mod, alias)) self.scope.add_import(Import(mod, alias))
freshscope = False freshscope = False
elif token == 'from': elif tok == 'from':
mod, token = self._parsedotname() mod, tok = self._parsedotname()
if not mod or token != "import": if not mod or tok != "import":
print "from: syntax error..." print "from: syntax error..."
continue continue
names = self._parseimportlist() names = self._parseimportlist()
for name, alias in names: for name, alias in names:
self.scope.add_import(Import(name, alias, mod)) self.scope.add_import(Import(name, alias, mod))
freshscope = False freshscope = False
elif tokentype == STRING: elif tokentype == tokenize.STRING:
if freshscope: self.scope.doc(token) if freshscope: self.scope.doc(tok)
elif tokentype == NAME: elif tokentype == tokenize.NAME:
name,token = self._parsedotname(token) name,tok = self._parsedotname(tok)
if token == '=': if tok == '=':
stmt = self._parseassignment() stmt = self._parseassignment()
dbg("parseassignment: %s = %s" % (name, stmt)) dbg("parseassignment: %s = %s" % (name, stmt))
if stmt != None: if stmt != None:
@@ -376,14 +313,10 @@ class PyFuzzyParser(object):
freshscope = False freshscope = False
except StopIteration: #thrown on EOF except StopIteration: #thrown on EOF
pass pass
except: #except:
dbg("parse error: %s, %s @ %s" % # dbg("parse error: %s, %s @ %s" %
(sys.exc_info()[0], sys.exc_info()[1], self.parserline)) # (sys.exc_info()[0], sys.exc_info()[1], self.parserline))
return self._adjustvisibility() return self.top #self._adjustvisibility()
def print_parser_code(self):
""" prints the result of the parser operation """
print self.scope.get_code()
def _sanitize(str): def _sanitize(str):
val = '' val = ''

4
test.py
View File

@@ -10,6 +10,10 @@ from token import OP as OP_TEST, INDENT as INDENT_TEST
aaa = 6; bbb = 13 aaa = 6; bbb = 13
ccc = bbb; d = open("test.py"); ccc = bbb; d = open("test.py");
class Intro:
def testing(self, string):
return string+","
class Supi(A, datetime.datetime): class Supi(A, datetime.datetime):
""" test123 """ """ test123 """
class 2 class 2