VimPlug/jedi-fork
1
0
Fork 0
forked from VimPlug/jedi
Code Pull Requests Activity

first version of statement parser

Browse Source
This commit is contained in:
David Halter
2012-02-26 04:27:00 +01:00
parent 166004565a
commit e83079d563
3 changed files with 100 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
pycomplete.py
View File

@@ -41,7 +41,7 @@ class Completer(object):
sc = self.parser.parse(text) sc = self.parser.parse(text)
self.sc = sc # TODO rm self.sc = sc # TODO rm
src = sc.get_code() src = sc.get_code()
dbg("source: %s" % src) #dbg("source: %s" % src)
#try: exec(src) in self.compldict #try: exec(src) in self.compldict
#except: dbg("parser: %s, %s" % (sys.exc_info()[0], sys.exc_info()[1])) #except: dbg("parser: %s, %s" % (sys.exc_info()[0], sys.exc_info()[1]))
#for l in sc.locals: #for l in sc.locals:
@@ -197,14 +197,14 @@ def showdbg():
text = cStringIO.StringIO(open('test.py').read()) text = cStringIO.StringIO(open('test.py').read())
cmpl = Completer() cmpl = Completer()
cmpl.evalsource(text, 51) cmpl.evalsource(text, 51)
print cmpl.sc.get_code() #print cmpl.sc.get_code()
all = cmpl.get_completions("cdef.", '') #all = cmpl.get_completions("cdef.", '')
print "Completions:", len(all) #print "Completions:", len(all)
for c in all: #for c in all:
print c['word'], # print c['word'],
print ',', # print ',',
print '' #print ''
showdbg() showdbg()
print cmpl.parser.top.get_code() print cmpl.parser.top.get_code()

118
pyfuzzyparser.py
View File

@@ -18,6 +18,7 @@ global is a special case and will not be used here
import sys import sys
import tokenize import tokenize
import cStringIO import cStringIO
import token
def indent_block(text, indention=" "): def indent_block(text, indention=" "):
@@ -110,7 +111,6 @@ class Class(Scope):
str += '(%s)' % ','.join(self.supers) str += '(%s)' % ','.join(self.supers)
str += ':\n' str += ':\n'
str += super(Class, self).get_code(True, indention) str += super(Class, self).get_code(True, indention)
print "get_code class %s %i" % (self.name, self.is_empty())
if self.is_empty(): if self.is_empty():
str += "pass\n" str += "pass\n"
return str return str
@@ -153,7 +153,7 @@ class Function(Scope):
str += super(Function, self).get_code(True, indention) str += super(Function, self).get_code(True, indention)
if self.is_empty(): if self.is_empty():
str += "pass\n" str += "pass\n"
print "func", self.locals #print "func", self.locals
return str return str
@@ -165,7 +165,9 @@ class Import(object):
2 2
:param line_nr: Line number. :param line_nr: Line number.
:param namespace: the namespace which is imported. :type line_nr: int
:param namespace: the import, as an array list, e.g. ['datetime', 'time']
:type namespace: list
:param alias: the alias (valid in the current namespace). :param alias: the alias (valid in the current namespace).
:param from_ns: from declaration in an import. :param from_ns: from declaration in an import.
:param star: if a star is used -> from time import *. :param star: if a star is used -> from time import *.
@@ -183,27 +185,29 @@ class Import(object):
self.star = star self.star = star
def get_code(self): def get_code(self):
ns = ".".join(self.namespace)
if self.alias: if self.alias:
ns_str = "%s as %s" % (self.namespace, self.alias) ns_str = "%s as %s" % (ns, self.alias)
else: else:
ns_str = self.namespace ns_str = ns
if self.from_ns: if self.from_ns:
if self.star: if self.star:
ns_str = '*' ns_str = '*'
return "test from %s import %s" % (self.from_ns, ns_str) return "from %s import %s" % (self.from_ns, ns_str)
else: else:
return "test import " + ns_str return "import " + ns_str
class Statement(object): class Statement(object):
""" This is the class for Local and Functions """ """
This is the class for Local and Functions
:param code:
:param locals:
"""
def __init__(self, code, locals, functions): def __init__(self, code, locals, functions):
""" self.code = code
@param line_nr self.locals = locals
@param stmt the statement string self.functions = functions
"""
self.line_nr = line_nr
self.stmt = stmt
def get_code(self): def get_code(self):
raise NotImplementedError() raise NotImplementedError()
@@ -271,7 +275,7 @@ class PyFuzzyParser(object):
if tokentype != tokenize.NAME and tok != '*': if tokentype != tokenize.NAME and tok != '*':
return ([], tok) return ([], tok)
else: else:
tok = pre_used_token tokentype, tok, indent = pre_used_token
names.append(tok) names.append(tok)
while True: while True:
tokentype, tok, indent = self.next() tokentype, tok, indent = self.next()
@@ -291,13 +295,14 @@ class PyFuzzyParser(object):
""" """
value_list = [] value_list = []
if pre_used_token: if pre_used_token:
n = self._parsedotname(pre_used_token) tokentype, tok, indent = pre_used_token
n = self._parsedotname(tok)
if n: if n:
value_list.append(n) value_list.append(n)
tokentype, tok, indent = self.next() tokentype, tok, indent = self.next()
while tok != 'in' or tokentype == tokenize.NEWLINE: while tok != 'in' and tokentype != tokenize.NEWLINE:
n = self._parsedotname(tok) n = self._parsedotname(self.current)
if n: if n:
value_list.append(n) value_list.append(n)
@@ -416,32 +421,79 @@ class PyFuzzyParser(object):
return "%s" % assign return "%s" % assign
def _parse_words(self, word): def _parse_words(self, pre_used_token):
""" """
Used to parse a word, if the tokenizer returned a word at the start of Used to parse a word, if the tokenizer returned a word at the start of
a new command. a new command.
:param pre_used_token: The pre parsed token.
:type pre_used_token: set
""" """
return return self._parse_statement(pre_used_token)
def _parse_statement(self, tok = None): def _parse_statement(self, pre_used_token = None):
""" """
Parses statements like: Parses statements like:
>>> a = test(b) >>> a = test(b)
>>> a += 3 - 2 or b >>> a += 3 - 2 or b
and so on. One row at a time. and so on. One row at a time.
:param pre_used_token: The pre parsed token.
:type pre_used_token: set
:return: Statement + last parsed token.
:rtype: (Statement, str)
""" """
string = tok string = ''
tok = True set_vars = []
while tok: used_funcs = []
tokentype, tok, indent = self.next() used_vars = []
token_type, tok, indent = pre_used_token
while tok != '\n' and tok != ';':
set_string = ''
print 'parse_stmt', tok, token.tok_name[token_type]
if token_type == tokenize.NAME:
if tok == 'pass':
set_string = ''
elif tok == 'return' or tok == 'del':
set_string = tok + ' '
elif tok == 'print':
set_string = ''
else:
path, tok = self._parsedotname(self.current)
if tok == '(':
# it must be a function
used_funcs.append(path)
else:
used_vars.append(path)
string += ".".join(path)
print 'parse_stmt', tok, token.tok_name[token_type]
if tok == '\n' or tok == ';':
break
if ('=' in tok and not tok in ['>=', '<=', '==', '!=']):
# there has been an assignement -> change vars
set_vars = used_vars
used_vars = []
if set_string:
string = set_string
else:
string += tok string += tok
return string token_type, tok, indent = self.next()
if not string:
return None, tok
print 'new_stat', string, set_vars, used_funcs, used_vars
#return Statement(), tok
def next(self): def next(self):
type, tok, position, dummy, self.parserline = self.gen.next() type, tok, position, dummy, self.parserline = self.gen.next()
(self.line_nr, indent) = position (self.line_nr, indent) = position
return (type, tok, indent) self.current = (type, tok, indent)
return self.current
def parse(self, text): def parse(self, text):
buf = cStringIO.StringIO(''.join(text) + '\n') buf = cStringIO.StringIO(''.join(text) + '\n')
@@ -451,7 +503,8 @@ class PyFuzzyParser(object):
try: try:
freshscope = True freshscope = True
while True: while True:
tokentype, tok, indent = self.next() full_token = self.next()
tokentype, tok, indent = full_token
dbg('main: tok=[%s] type=[%s] indent=[%s]'\ dbg('main: tok=[%s] type=[%s] indent=[%s]'\
% (tok, tokentype, indent)) % (tok, tokentype, indent))
@@ -491,6 +544,7 @@ class PyFuzzyParser(object):
freshscope = False freshscope = False
#loops #loops
elif tok == 'for': elif tok == 'for':
print tok, tokentype
value_list, tok = self._parse_value_list() value_list, tok = self._parse_value_list()
if tok == 'in': if tok == 'in':
statement, tok = self._parse_statement() statement, tok = self._parse_statement()
@@ -499,10 +553,14 @@ class PyFuzzyParser(object):
elif tok == 'while': elif tok == 'while':
param_list = self._parse_while_loop() param_list = self._parse_while_loop()
elif tok == 'global':
self._parse_words(full_token)
elif tokentype == tokenize.STRING: elif tokentype == tokenize.STRING:
if freshscope: if freshscope:
self.scope.doc(tok) self.scope.doc(tok)
elif tokentype == tokenize.NAME: elif tokentype == tokenize.NAME:
self._parse_words(full_token)
"""
name, tok = self._parsedotname(tok) name, tok = self._parsedotname(tok)
if tok == '=': if tok == '=':
stmt = self._parseassignment() stmt = self._parseassignment()
@@ -512,6 +570,7 @@ class PyFuzzyParser(object):
else: else:
#print "_not_implemented_", tok, self.parserline #print "_not_implemented_", tok, self.parserline
pass pass
"""
freshscope = False freshscope = False
#else: #else:
#print "_not_implemented_", tok, self.parserline #print "_not_implemented_", tok, self.parserline
@@ -537,4 +596,5 @@ def _sanitize(str):
def dbg(*args): def dbg(*args):
print args #print args
pass

3
test.py
View File

@@ -56,7 +56,8 @@ def ass_test(a):
a -= 1 a -= 1
b, c, d = (1,2,3) b, c, d = (1,2,3)
del b del b
return (a+b)*2 result = int((a+b)*2)
return result
matrix = [[1,2,3], [4,5,6], [7,8,9]] matrix = [[1,2,3], [4,5,6], [7,8,9]]