forked from VimPlug/jedi
664 lines
22 KiB
Python
664 lines
22 KiB
Python
""""
|
|
TODO Description: This is a parser
|
|
|
|
TODO be tolerant with indents
|
|
TODO dictionaries not working with statement parser
|
|
TODO except has local vars
|
|
TODO take special care for future imports
|
|
|
|
scope
|
|
imports
|
|
subscopes
|
|
statements
|
|
|
|
Ignored statements:
|
|
- print (no use for it)
|
|
- exec (dangerous - not controllable)
|
|
|
|
global is a special case and will not be used here
|
|
"""
|
|
import sys
|
|
import tokenize
|
|
import cStringIO
|
|
import token
|
|
import re
|
|
|
|
|
|
def indent_block(text, indention=" "):
|
|
""" This function indents a text block with a default of four spaces """
|
|
temp = ''
|
|
while text and text[-1] == '\n':
|
|
temp += text[-1]
|
|
text = text[:-1]
|
|
lines = text.split('\n')
|
|
return '\n'.join(map(lambda s: indention + s, lines)) + temp
|
|
|
|
|
|
class Scope(object):
|
|
def __init__(self, name, indent, line_nr, docstr=''):
|
|
self.subscopes = []
|
|
self.locals = []
|
|
self.imports = []
|
|
self.statements = []
|
|
self.docstr = docstr
|
|
self.parent = None
|
|
self.name = name
|
|
self.indent = indent
|
|
self.line_nr = line_nr
|
|
|
|
def add_scope(self, sub):
|
|
# print 'push scope: [%s@%s]' % (sub.name, sub.indent)
|
|
sub.parent = self
|
|
self.subscopes.append(sub)
|
|
return sub
|
|
|
|
def add_statement(self, stmt):
|
|
"""
|
|
Used to add a Statement or a Scope.
|
|
A statement would be a normal command (Statement) or a Scope (Flow).
|
|
"""
|
|
if isinstance(stmt, Scope):
|
|
stmt.parent = self
|
|
self.statements.append(stmt)
|
|
return stmt
|
|
|
|
def doc(self, str):
|
|
""" Clean up a docstring """
|
|
d = str.replace('\n', ' ')
|
|
d = d.replace('\t', ' ')
|
|
while d.find(' ') > -1:
|
|
d = d.replace(' ', ' ')
|
|
while d[0] in '"\'\t ':
|
|
d = d[1:]
|
|
while d[-1] in '"\'\t ':
|
|
d = d[:-1]
|
|
dbg("Scope(%s)::docstr = %s" % (self, d))
|
|
self.docstr = d
|
|
|
|
def add_local(self, loc):
|
|
self.locals.append(loc)
|
|
|
|
def add_import(self, imp):
|
|
self.imports.append(imp)
|
|
|
|
def _checkexisting(self, test):
|
|
"Convienance function... keep out duplicates"
|
|
if test.find('=') > -1:
|
|
var = test.split('=')[0].strip()
|
|
for l in self.locals:
|
|
if l.find('=') > -1 and var == l.split('=')[0].strip():
|
|
self.locals.remove(l)
|
|
|
|
def get_code(self, first_indent=False, indention=" "):
|
|
""" Returns the code of the current scope. """
|
|
string = ""
|
|
if len(self.docstr) > 0:
|
|
string += '"""' + self.docstr + '"""\n'
|
|
for i in self.imports:
|
|
string += i.get_code()
|
|
for sub in self.subscopes:
|
|
#string += str(sub.line_nr)
|
|
string += sub.get_code(first_indent=True, indention=indention)
|
|
for stmt in self.statements:
|
|
string += stmt.get_code()
|
|
|
|
if first_indent:
|
|
string = indent_block(string, indention=indention)
|
|
return string
|
|
|
|
def is_empty(self):
|
|
"""
|
|
this function returns true if there are no subscopes, imports, locals.
|
|
"""
|
|
return not (self.locals or self.imports or self.subscopes or \
|
|
self.statements)
|
|
|
|
|
|
class Class(Scope):
|
|
def __init__(self, name, supers, indent, line_nr, docstr=''):
|
|
super(Class, self).__init__(name, indent, line_nr, docstr)
|
|
self.supers = supers
|
|
|
|
def get_code(self, first_indent=False, indention=" "):
|
|
str = 'class %s' % (self.name)
|
|
if len(self.supers) > 0:
|
|
str += '(%s)' % ','.join(self.supers)
|
|
str += ':\n'
|
|
str += super(Class, self).get_code(True, indention)
|
|
if self.is_empty():
|
|
str += "pass\n"
|
|
return str
|
|
|
|
|
|
class Flow(Scope):
|
|
"""
|
|
Used to describe programming structure - flow statements,
|
|
which indent code, but are not classes or functions:
|
|
|
|
- for
|
|
- while
|
|
- if
|
|
- try
|
|
- with
|
|
|
|
Therefore statements like else, except and finally are also here,
|
|
they are now saved in the root flow elements, but in the next variable.
|
|
|
|
:param command: The flow command, if, while, else, etc.
|
|
:type command: str
|
|
:param statement: The statement after the flow comand -> while 'statement'.
|
|
:type statement: Statement
|
|
:param indent: The indent level of the flow statement.
|
|
:type indent: int
|
|
:param line_nr: Line number of the flow statement.
|
|
:type line_nr: int
|
|
:param set_args: Local variables used in the for loop (only there).
|
|
:type set_args: list
|
|
"""
|
|
def __init__(self, command, statement, indent, line_nr, set_args=None):
|
|
name = "%s@%s" % (command, line_nr)
|
|
super(Flow, self).__init__(name, indent, line_nr, '')
|
|
self.command = command
|
|
self.statement = statement
|
|
self.set_args = set_args
|
|
self.next = None
|
|
|
|
def get_code(self, first_indent=False, indention=" "):
|
|
if self.set_args:
|
|
args = ",".join(map(lambda x: x.get_code(), self.set_args))
|
|
args += ' in '
|
|
else:
|
|
args = ''
|
|
|
|
if self.statement:
|
|
stmt = self.statement.get_code(new_line=False)
|
|
else:
|
|
stmt = ''
|
|
str = "%s %s%s:\n" % (self.command, args, stmt)
|
|
str += super(Flow, self).get_code(True, indention)
|
|
if self.next:
|
|
str += self.next.get_code()
|
|
return str
|
|
|
|
def set_next(self, next):
|
|
""" Set the next element in the flow, those are else, except, etc. """
|
|
if self.next:
|
|
return self.next.set_next(next)
|
|
else:
|
|
self.next = next
|
|
next.parent = self.parent
|
|
return next
|
|
|
|
|
|
class Function(Scope):
|
|
def __init__(self, name, params, indent, line_nr, docstr=''):
|
|
Scope.__init__(self, name, indent, line_nr, docstr)
|
|
self.params = params
|
|
|
|
def get_code(self, first_indent=False, indention=" "):
|
|
str = "def %s(%s):\n" % (self.name, ','.join(self.params))
|
|
str += super(Function, self).get_code(True, indention)
|
|
if self.is_empty():
|
|
str += "pass\n"
|
|
return str
|
|
|
|
|
|
class Import(object):
|
|
"""
|
|
stores the imports of any scopes.
|
|
|
|
>>> 1+1
|
|
2
|
|
|
|
:param line_nr: Line number.
|
|
:type line_nr: int
|
|
:param namespace: the import, as an array list, e.g. ['datetime', 'time']
|
|
:type namespace: list
|
|
:param alias: the alias (valid in the current namespace).
|
|
:param from_ns: from declaration in an import.
|
|
:param star: if a star is used -> from time import *.
|
|
|
|
:returns: test
|
|
:raises:
|
|
|
|
TODO check star?
|
|
"""
|
|
def __init__(self, line_nr, namespace, alias='', from_ns='', star=False):
|
|
self.line_nr = line_nr
|
|
self.namespace = namespace
|
|
self.alias = alias
|
|
self.from_ns = from_ns
|
|
self.star = star
|
|
|
|
def get_code(self):
|
|
if self.alias:
|
|
ns_str = "%s as %s" % (self.namespace, self.alias)
|
|
else:
|
|
ns_str = str(self.namespace)
|
|
if self.from_ns:
|
|
if self.star:
|
|
ns_str = '*'
|
|
return "from %s import %s" % (self.from_ns, ns_str) + '\n'
|
|
else:
|
|
return "import " + ns_str + '\n'
|
|
|
|
|
|
class Statement(object):
|
|
"""
|
|
This is the class for all different statements.
|
|
:param code:
|
|
:param locals:
|
|
"""
|
|
def __init__(self, code, set_vars, used_funcs, used_vars, indent, line_nr):
|
|
self.code = code
|
|
self.set_vars = set_vars
|
|
self.used_funcs = used_funcs
|
|
self.used_vars = used_vars
|
|
|
|
self.indent = indent
|
|
self.line_nr = line_nr
|
|
|
|
def get_code(self, new_line=True):
|
|
if new_line:
|
|
return self.code + '\n'
|
|
else:
|
|
return self.code
|
|
|
|
|
|
class Name(object):
|
|
"""
|
|
Used to define names in python.
|
|
Which means the whole namespace/class/function stuff.
|
|
So a name like "module.class.function"
|
|
would result in an array of [module, class, function]
|
|
"""
|
|
def __init__(self, names, indent, line_nr):
|
|
super(Name, self).__init__()
|
|
self.names = names
|
|
self.indent = indent
|
|
self.line_nr = line_nr
|
|
|
|
def get_code(self):
|
|
""" Returns the names in a full string format """
|
|
return ".".join(self.names)
|
|
|
|
def __str__(self):
|
|
return self.get_code()
|
|
|
|
|
|
class PyFuzzyParser(object):
|
|
"""
|
|
This class is used to parse a Python file, it then divides them into a
|
|
class structure of differnt scopes.
|
|
"""
|
|
def __init__(self):
|
|
self.top = Scope('global', 0, 0)
|
|
self.scope = self.top
|
|
self.current = (None, None, None)
|
|
|
|
def _parsedotname(self, pre_used_token=None):
|
|
"""
|
|
The dot name parser parses a name, variable or function and returns
|
|
their names.
|
|
:return: list of the names, token_type, nexttoken, start_indent.
|
|
:rtype: (Name, int, str, int)
|
|
"""
|
|
names = []
|
|
if pre_used_token is None:
|
|
token_type, tok, indent = self.next()
|
|
if token_type != tokenize.NAME and tok != '*':
|
|
return ([], tok)
|
|
else:
|
|
token_type, tok, indent = pre_used_token
|
|
names.append(tok)
|
|
start_indent = indent
|
|
while True:
|
|
token_type, tok, indent = self.next()
|
|
if tok != '.':
|
|
break
|
|
token_type, tok, indent = self.next()
|
|
if token_type != tokenize.NAME:
|
|
break
|
|
names.append(tok)
|
|
return (names, token_type, tok, start_indent)
|
|
|
|
def _parse_value_list(self, pre_used_token=None):
|
|
"""
|
|
A value list is a comma separated list. This is used for:
|
|
>>> for a,b,self.c in enumerate(test)
|
|
"""
|
|
value_list = []
|
|
if pre_used_token:
|
|
token_type, tok, indent = pre_used_token
|
|
n, token_type, tok, start_indent = self._parsedotname(tok)
|
|
if n:
|
|
value_list.append(Name(n, start_indent, self.line_nr))
|
|
|
|
token_type, tok, indent = self.next()
|
|
while tok != 'in' and token_type != tokenize.NEWLINE:
|
|
n, token_type, tok, start_indent = self._parsedotname(self.current)
|
|
if n:
|
|
value_list.append(Name(n, start_indent, self.line_nr))
|
|
if tok == 'in':
|
|
break
|
|
|
|
print 'for_tok', tok
|
|
token_type, tok, indent = self.next()
|
|
return (value_list, tok)
|
|
|
|
def _parseimportlist(self):
|
|
imports = []
|
|
while True:
|
|
name, token_type, tok, start_indent = self._parsedotname()
|
|
if not name:
|
|
break
|
|
name2 = None
|
|
if tok == 'as':
|
|
name2, token_type, tok, start_indent2 = self._parsedotname()
|
|
name2 = Name(name2, start_indent2, self.line_nr)
|
|
imports.append((Name(name, start_indent, self.line_nr), name2))
|
|
while tok != "," and "\n" not in tok:
|
|
token_type, tok, indent = self.next()
|
|
if tok != ",":
|
|
break
|
|
return imports
|
|
|
|
def _parseparen(self):
|
|
name = ''
|
|
names = []
|
|
level = 1
|
|
while True:
|
|
token_type, tok, indent = self.next()
|
|
if tok in (')', ',') and level == 1:
|
|
if '=' not in name:
|
|
name = name.replace(' ', '')
|
|
names.append(name.strip())
|
|
name = ''
|
|
if tok == '(':
|
|
level += 1
|
|
name += "("
|
|
elif tok == ')':
|
|
level -= 1
|
|
if level == 0:
|
|
break
|
|
else:
|
|
name += ")"
|
|
elif tok == ',' and level == 1:
|
|
pass
|
|
else:
|
|
name += "%s " % str(tok)
|
|
return names
|
|
|
|
def _parsefunction(self, indent):
|
|
token_type, fname, ind = self.next()
|
|
if token_type != tokenize.NAME:
|
|
return None
|
|
|
|
token_type, open, ind = self.next()
|
|
if open != '(':
|
|
return None
|
|
params = self._parseparen()
|
|
|
|
token_type, colon, ind = self.next()
|
|
if colon != ':':
|
|
return None
|
|
|
|
return Function(fname, params, indent, self.line_nr)
|
|
|
|
def _parseclass(self, indent):
|
|
token_type, cname, ind = self.next()
|
|
if token_type != tokenize.NAME:
|
|
print "class: syntax error - token is not a name@%s (%s: %s)" \
|
|
% (self.line_nr, token.tok_name[token_type], cname)
|
|
return None
|
|
|
|
super = []
|
|
token_type, next, ind = self.next()
|
|
if next == '(':
|
|
super = self._parseparen()
|
|
elif next != ':':
|
|
print "class: syntax error - %s@%s" % (cname, self.line_nr)
|
|
return None
|
|
|
|
return Class(cname, super, indent, self.line_nr)
|
|
|
|
def _parseassignment(self):
|
|
assign = ''
|
|
token_type, tok, indent = self.next()
|
|
if token_type == tokenize.STRING or tok == 'str':
|
|
return '""'
|
|
elif tok == '(' or tok == 'tuple':
|
|
return '()'
|
|
elif tok == '[' or tok == 'list':
|
|
return '[]'
|
|
elif tok == '{' or tok == 'dict':
|
|
return '{}'
|
|
elif token_type == tokenize.NUMBER:
|
|
return '0'
|
|
elif tok == 'open' or tok == 'file':
|
|
return 'file'
|
|
elif tok == 'None':
|
|
return '_PyCmplNoType()'
|
|
elif tok == 'type':
|
|
return 'type(_PyCmplNoType)' # only for method resolution
|
|
else:
|
|
assign += tok
|
|
level = 0
|
|
while True:
|
|
token_type, tok, indent = self.next()
|
|
if tok in ('(', '{', '['):
|
|
level += 1
|
|
elif tok in (']', '}', ')'):
|
|
level -= 1
|
|
if level == 0:
|
|
break
|
|
elif level == 0:
|
|
if tok in (';', '\n'):
|
|
break
|
|
assign += tok
|
|
return "%s" % assign
|
|
|
|
def _parse_statement(self, pre_used_token=None):
|
|
"""
|
|
Parses statements like:
|
|
|
|
>>> a = test(b)
|
|
>>> a += 3 - 2 or b
|
|
|
|
and so on. One row at a time.
|
|
|
|
:param pre_used_token: The pre parsed token.
|
|
:type pre_used_token: set
|
|
:return: Statement + last parsed token.
|
|
:rtype: (Statement, str)
|
|
"""
|
|
string = ''
|
|
set_vars = []
|
|
used_funcs = []
|
|
used_vars = []
|
|
|
|
if pre_used_token:
|
|
token_type, tok, indent = pre_used_token
|
|
else:
|
|
token_type, tok, indent = self.next()
|
|
|
|
is_break_token = lambda tok: tok in ['\n', ':', ';']
|
|
|
|
while not is_break_token(tok):
|
|
set_string = ''
|
|
#print 'parse_stmt', tok, token.tok_name[token_type]
|
|
if token_type == tokenize.NAME:
|
|
print 'is_name', tok
|
|
if tok == 'pass':
|
|
set_string = ''
|
|
elif tok in ['return', 'yield', 'del', 'raise', 'assert']:
|
|
set_string = tok + ' '
|
|
elif tok == 'print':
|
|
set_string = tok + ' '
|
|
else:
|
|
path, token_type, tok, start_indent = \
|
|
self._parsedotname(self.current)
|
|
print 'path', path
|
|
n = Name(path, start_indent, self.line_nr)
|
|
if tok == '(':
|
|
# it must be a function
|
|
used_funcs.append(n)
|
|
else:
|
|
used_vars.append(n)
|
|
if string:
|
|
print 'str', string[-1]
|
|
if string and re.match(r'[\w\d]', string[-1]):
|
|
print 'yay'
|
|
string += ' '
|
|
#if token_type == tokenize.NAME \
|
|
# and self.last_token[0] == tokenize.NAME:
|
|
# print 'last_token', self.last_token, token_type
|
|
# string += ' ' + tok
|
|
string += ".".join(path)
|
|
#print 'parse_stmt', tok, token.tok_name[token_type]
|
|
continue
|
|
elif ('=' in tok and not tok in ['>=', '<=', '==', '!=']):
|
|
# there has been an assignement -> change vars
|
|
set_vars = used_vars
|
|
used_vars = []
|
|
|
|
if set_string:
|
|
string = set_string
|
|
else:
|
|
string += tok
|
|
# caution: don't use indent anywhere,
|
|
# it's not working with the name parsing
|
|
token_type, tok, indent_dummy = self.next()
|
|
if not string:
|
|
return None, tok
|
|
#print 'new_stat', string, set_vars, used_funcs, used_vars
|
|
stmt = Statement(string, set_vars, used_funcs, used_vars,\
|
|
self.line_nr, indent)
|
|
return stmt, tok
|
|
|
|
def next(self):
|
|
type, tok, position, dummy, self.parserline = self.gen.next()
|
|
(self.line_nr, indent) = position
|
|
self.last_token = self.current
|
|
self.current = (type, tok, indent)
|
|
return self.current
|
|
|
|
def parse(self, text):
|
|
"""
|
|
The main part of the program. It analyzes the given code-text and
|
|
returns a tree-like scope. For a more detailed description, see the
|
|
class description.
|
|
"""
|
|
buf = cStringIO.StringIO(''.join(text) + '\n')
|
|
self.gen = tokenize.generate_tokens(buf.readline)
|
|
self.currentscope = self.scope
|
|
|
|
try:
|
|
extended_flow = ['else', 'except', 'finally']
|
|
statement_toks = ['{', '[', '(', '`']
|
|
|
|
freshscope = True
|
|
while True:
|
|
token_type, tok, indent = self.next()
|
|
dbg('main: tok=[%s] type=[%s] indent=[%s]'\
|
|
% (tok, token_type, indent))
|
|
|
|
if token_type == tokenize.DEDENT:
|
|
print 'dedent', self.scope.name
|
|
self.scope = self.scope.parent
|
|
elif tok == 'def':
|
|
func = self._parsefunction(indent)
|
|
if func is None:
|
|
print "function: syntax error@%s" % self.line_nr
|
|
continue
|
|
dbg("new scope: function %s" % (func.name))
|
|
freshscope = True
|
|
self.scope = self.scope.add_scope(func)
|
|
elif tok == 'class':
|
|
cls = self._parseclass(indent)
|
|
if cls is None:
|
|
continue
|
|
freshscope = True
|
|
dbg("new scope: class %s" % (cls.name))
|
|
self.scope = self.scope.add_scope(cls)
|
|
# import stuff
|
|
elif tok == 'import':
|
|
imports = self._parseimportlist()
|
|
for mod, alias in imports:
|
|
self.scope.add_import(Import(self.line_nr, mod, alias))
|
|
freshscope = False
|
|
elif tok == 'from':
|
|
mod, token_type, tok, start_indent = self._parsedotname()
|
|
if not mod or tok != "import":
|
|
print "from: syntax error..."
|
|
continue
|
|
mod = Name(mod, start_indent, self.line_nr)
|
|
names = self._parseimportlist()
|
|
for name, alias in names:
|
|
i = Import(self.line_nr, name, alias, mod)
|
|
self.scope.add_import(i)
|
|
freshscope = False
|
|
#loops
|
|
elif tok == 'for':
|
|
value_list, tok = self._parse_value_list()
|
|
if tok == 'in':
|
|
statement, tok = self._parse_statement()
|
|
if tok == ':':
|
|
f = Flow('for', statement, indent, self.line_nr, \
|
|
value_list)
|
|
dbg("new scope: flow %s" % (f.name))
|
|
self.scope = self.scope.add_statement(f)
|
|
|
|
elif tok in ['if', 'while', 'try', 'with'] + extended_flow:
|
|
# TODO with statement has local variables
|
|
command = tok
|
|
statement, tok = self._parse_statement()
|
|
if tok == ':':
|
|
f = Flow(command, statement, indent, self.line_nr)
|
|
dbg("new scope: flow %s" % (f.name))
|
|
if command in extended_flow:
|
|
# the last statement has to be another part of
|
|
# the flow statement
|
|
self.scope = self.scope.statements[-1].set_next(f)
|
|
else:
|
|
self.scope = self.scope.add_statement(f)
|
|
|
|
elif tok == 'global':
|
|
self._parse_statement(self.current)
|
|
pass
|
|
# TODO add suport for global
|
|
elif token_type == tokenize.STRING:
|
|
if freshscope:
|
|
self.scope.doc(tok)
|
|
elif token_type == tokenize.NAME or tok in statement_toks:
|
|
stmt, tok = self._parse_statement(self.current)
|
|
if stmt:
|
|
self.scope.add_statement(stmt)
|
|
freshscope = False
|
|
#else:
|
|
#print "_not_implemented_", tok, self.parserline
|
|
except StopIteration: # thrown on EOF
|
|
pass
|
|
#except:
|
|
# dbg("parse error: %s, %s @ %s" %
|
|
# (sys.exc_info()[0], sys.exc_info()[1], self.parserline))
|
|
return self.top
|
|
|
|
|
|
def _sanitize(str):
|
|
val = ''
|
|
level = 0
|
|
for c in str:
|
|
if c in ('(', '{', '['):
|
|
level += 1
|
|
elif c in (']', '}', ')'):
|
|
level -= 1
|
|
elif level == 0:
|
|
val += c
|
|
return val
|
|
|
|
|
|
def dbg(*args):
|
|
#print args
|
|
pass
|