diff --git a/builtin.py b/builtin.py index ddd0b2f4..a87f1285 100644 --- a/builtin.py +++ b/builtin.py @@ -1,10 +1,23 @@ import re +import sys +import os import debug import parsing class Parser(object): + """ + This module is a parser for all builtin modules, which are programmed in + C/C++. It should also work on third party modules. + It can be instantiated with either a path or a name of the module. The path + is important for third party modules. + + :param name: The name of the module. + :param path: The path of the module. + :param sys_path: The sys.path, which is can be customizable. + """ + map_types = { 'floating point number': '0.0', 'string': '""', @@ -17,37 +30,55 @@ class Parser(object): 'object': '{}', # TODO things like dbg: ('not working', 'tuple of integers') } + cache = {} - """ This module tries to imitate parsing.Scope """ - def __init__(self, name): - self.name = name + def __init__(self, name=None, path=None, sys_path=sys.path): + self.path = path + print path + if name: + self.name = name + else: + name = os.path.basename(self.path) + self.name = name.rpartition('.')[0] # cut file type (normally .so) + self.path = os.path.dirname(self.path) + print self.name, self.path self._content = {} self._parser = None self._module = None + self.sys_path = sys_path @property def module(self): if not self._module: - print 'import', self.name + self.sys_path.insert(0, self.path) + + temp, sys.path = sys.path, self.sys_path + print 'sypa', sys.path exec 'import %s as module' % self.name in self._content - print 'import2', self.name + self.sys_path, sys.path = sys.path, temp + + self.sys_path.pop(0) self._module = self._content['module'] + print 'mod', self._content['module'] return self._module @property def parser(self): """ get the parser lazy """ - if self._parser: - return self._parser - else: - code = self._generate_code(self.module) + if not self._parser: try: - self._parser = parsing.PyFuzzyParser(code) - except: - debug.warning('not possible to resolve', self.name, code) - #open('builtin_fail', 'w').write(code) - raise - return self._parser + self._parser = Parser.cache[self.name, self.path].parser + except KeyError: + code = self._generate_code(self.module) + try: + self._parser = parsing.PyFuzzyParser(code) + except: + debug.warning('not possible to resolve', self.name, code) + #open('builtin_fail', 'w').write(code) + raise + else: + Parser.cache[self.name, self.path] = self + return self._parser def _generate_code(self, scope, depth=0): """ @@ -133,8 +164,8 @@ class Parser(object): if depth == 0: #with open('writeout.py', 'w') as f: # f.write(code) - import sys - sys.stdout.write(code) + #import sys + #sys.stdout.write(code) #exit() pass return code @@ -195,21 +226,9 @@ def parse_function_doc(func): ret = 'return ' + ret return param_str, ret -"""if current.arr_type == parsing.Array.EMPTY: - # the normal case - no array type - debug.dbg('length', len(current)) -elif current.arr_type == parsing.Array.LIST: - result.append(__builtin__.list()) -elif current.arr_type == parsing.Array.SET: - result.append(__builtin__.set()) -elif current.arr_type == parsing.Array.TUPLE: - result.append(__builtin__.tuple()) -elif current.arr_type == parsing.Array.DICT: - result.append(__builtin__.dict()) - """ class _Builtin(object): - _builtins = Parser('__builtin__') + _builtins = Parser(name='__builtin__') @property def scope(self): diff --git a/evaluate.py b/evaluate.py index d72f8598..4f8281f5 100644 --- a/evaluate.py +++ b/evaluate.py @@ -270,22 +270,17 @@ def get_scopes_for_name(scope, name, search_global=False): def strip_imports(scopes): """ - Here we strip the imports - they don't get resolved necessarily, but star - imports are looked at here. + Here we strip the imports - they don't get resolved necessarily. + Really used anymore? """ result = [] for s in scopes: if isinstance(s, parsing.Import): print 'dini mueter, steile griech!' try: - new_scopes = follow_import(s) + result += follow_import(s) except modules.ModuleNotFound: - debug.dbg('Module not found: ' + str(s)) - else: - result += new_scopes - for n in new_scopes: - result += strip_imports(i for i in n.get_imports() - if i.star) + debug.warning('Module not found: ' + str(s)) else: result.append(s) return result @@ -415,5 +410,10 @@ def follow_import(_import): else: scopes = [scope] + for s in scopes: + scopes += strip_imports(i for i in s.get_imports() if i.star) + debug.dbg('after import', scopes, rest) - return scopes + + # filter duplicate modules + return list(set(scopes)) diff --git a/ftest.py b/ftest.py index 357c966c..2f53778e 100755 --- a/ftest.py +++ b/ftest.py @@ -2,9 +2,9 @@ import functions -functions.debug.debug_function = functions.debug.print_to_stdout +#functions.debug.debug_function = functions.debug.print_to_stdout functions.debug.ignored_modules = ['parsing', 'builtin'] -#functions.debug.ignored_modules = ['parsing', 'builtin', 'evaluate', 'modules'] +functions.debug.ignored_modules = ['parsing', 'builtin', 'evaluate', 'modules'] functions.modules.module_find_path.insert(0, '.') f_name = 'parsetest.py' @@ -16,4 +16,5 @@ code = f.read() for i in range(1): completions = functions.complete(code, 150, 200, path) -print '\n', ', '.join(str(c) for c in completions) +#print '\n', ', '.join(sorted(str(c) for c in completions)) +print '#', len(completions) diff --git a/modules.py b/modules.py index 3b72cbfe..2a9a91f1 100644 --- a/modules.py +++ b/modules.py @@ -11,7 +11,6 @@ load_module_cb = None module_find_path = sys.path[1:] - class ModuleNotFound(Exception): pass @@ -42,7 +41,7 @@ class File(object): try: timestamp, _parser = File.module_cache[self.module_path] if timestamp == os.path.getmtime(self.module_path): - debug.dbg('hit cache') + debug.dbg('hit module cache') return _parser except: pass @@ -81,15 +80,12 @@ def find_module(current_module, point_path): i = imp.find_module(string, path) except ImportError: # find builtins (ommit path): - i = imp.find_module(string) - if i[0]: - # if the import has a file descriptor, it cannot be a builtin. - raise + i = imp.find_module(string, module_find_path) return i # TODO handle relative paths - they are included in the import object current_namespace = None - sys.path.insert(0, os.path.dirname(current_module.module_path)) + module_find_path.insert(0, os.path.dirname(current_module.module_path)) # now execute those paths rest = [] for i, s in enumerate(point_path): @@ -102,7 +98,7 @@ def find_module(current_module, point_path): raise ModuleNotFound( 'The module you searched has not been found') - sys.path.pop(0) + module_find_path.pop(0) path = current_namespace[1] is_package_directory = current_namespace[2][2] == imp.PKG_DIRECTORY @@ -119,9 +115,9 @@ def find_module(current_module, point_path): source = current_namespace[0].read() if path.endswith('.py'): f = File(path, source) - if not f: - print 'lala' - f = builtin.Parser(path) - print 'lala2' + else: + f = builtin.Parser(path=path) + else: + f = builtin.Parser(name=path) return f.parser.top, rest diff --git a/parsing.py b/parsing.py index 00cbc3c3..a3b45020 100644 --- a/parsing.py +++ b/parsing.py @@ -145,7 +145,7 @@ class Scope(Simple): def get_imports(self): """ Gets also the imports within flow statements """ - i = self.imports + i = [] + self.imports for s in self.statements: if isinstance(s, Scope): i += s.get_imports() @@ -404,6 +404,12 @@ class Flow(Scope): else: return self.get_parent_until(Class, Function).get_set_vars() + def get_imports(self): + i = super(Flow, self).get_imports() + if self.next: + i += self.next.get_imports() + return i + def set_next(self, next): """ Set the next element in the flow, those are else, except, etc. """ if self.next: @@ -1199,7 +1205,7 @@ class PyFuzzyParser(object): while indent <= self.scope.indent \ and (token_type == tokenize.NAME or tok in ['(', '['])\ and self.scope != self.top: - debug.warning('syntax error: dedent @%s - %s<=%s', \ + debug.dbg('syntax: dedent @%s - %s<=%s', \ (self.line_nr, indent, self.scope.indent)) self.scope.line_end = self.line_nr self.scope = self.scope.parent