From e779cc8c97a88fd5da40161b0612ba524df6b92d Mon Sep 17 00:00:00 2001 From: David Halter Date: Mon, 16 Jul 2012 02:19:48 +0200 Subject: [PATCH] basic imports rewriting, which has still it's rough edges --- evaluate.py | 67 +++----------------------------------- functions.py | 32 ++++++++++-------- modules.py | 64 ------------------------------------ parsing.py | 53 ++++++++++++++++++++++++------ test/completion/imports.py | 10 ++++++ test/run.py | 4 +-- 6 files changed, 79 insertions(+), 151 deletions(-) diff --git a/evaluate.py b/evaluate.py index 8117e2ec..0128bc8b 100644 --- a/evaluate.py +++ b/evaluate.py @@ -30,6 +30,7 @@ import parsing import modules import debug import builtin +import imports import helpers memoize_caches = [] @@ -421,7 +422,7 @@ class Execution(Executable): debug.dbg('exec result: %s in %s' % (stmts, self)) - return strip_imports(stmts) + return imports.strip_imports(stmts) def _get_function_returns(self, evaluate_generator): func = self.base @@ -815,7 +816,7 @@ def get_names_for_scope(scope, position=None, star_search=True): # Add star imports. if star_search: - for s in remove_star_imports(start_scope.get_parent_until()): + for s in imports.remove_star_imports(start_scope.get_parent_until()): for g in get_names_for_scope(s, star_search=False): yield g @@ -938,23 +939,6 @@ def get_scopes_for_name(scope, name_str, position=None, search_global=False): return remove_statements(filter_name(scope_generator)) -def strip_imports(scopes): - """ - Here we strip the imports - they don't get resolved necessarily. - Really used anymore? - """ - result = [] - for s in scopes: - if isinstance(s, parsing.Import): - try: - result += follow_import(s) - except modules.ModuleNotFound: - debug.warning('Module not found: ' + str(s)) - else: - result.append(s) - return result - - def assign_tuples(tup, results, seek_name): """ This is a normal assignment checker. In python functions and other things @@ -1099,7 +1083,7 @@ def follow_call(call): # This is the first global lookup. scopes = get_scopes_for_name(scope, current, position=position, search_global=True) - result = strip_imports(scopes) + result = imports.strip_imports(scopes) if result != scopes: # Reset the position, when imports where stripped. @@ -1162,49 +1146,8 @@ def follow_path(path, scope, position=None): else: # TODO Check magic class methods and return them also. # This is the typical lookup while chaining things. - result = strip_imports(get_scopes_for_name(scope, current, + result = imports.strip_imports(get_scopes_for_name(scope, current, position=position)) return follow_paths(path, result, position=position) -def follow_import(_import): - """ - follows a module name and returns the parser. - :param _import: The import statement. - :type _import: parsing.Import - """ - # Set path together. - ns_list = [] - if _import.from_ns: - ns_list += _import.from_ns.names - if _import.namespace: - ns_list += _import.namespace.names - - loaded_in = _import.get_parent_until() - - scope, rest = modules.find_module(loaded_in, ns_list) - if rest: - scopes = follow_path(iter(rest), scope) - else: - scopes = [scope] - - new = [] - for scope in scopes: - new += remove_star_imports(scope) - scopes += new - - debug.dbg('after import', scopes, rest) - return scopes - - -def remove_star_imports(scope): - """ - """ - modules = strip_imports(i for i in scope.get_imports() if i.star) - new = [] - for m in modules: - new += remove_star_imports(m) - modules += new - - # Filter duplicate modules. - return set(modules) diff --git a/functions.py b/functions.py index b8c3f392..0bdfeb63 100644 --- a/functions.py +++ b/functions.py @@ -5,6 +5,7 @@ import parsing import evaluate import modules import debug +import imports __all__ = ['complete', 'get_completion_parts', 'get_definitions', 'set_debug_function'] @@ -183,20 +184,25 @@ def prepare_goto(source, position, source_path, is_like_search): debug.dbg('start: %s in %s' % (path, scope)) - # just parse one statement, take it and evaluate it - r = parsing.PyFuzzyParser(path, source_path) - try: - stmt = r.top.statements[0] - except IndexError: - if is_like_search: - path_tuple = path, dot, like - else: - path_tuple = () - raise NotFoundError(scope, path_tuple) + user_stmt = f.parser.user_stmt + if isinstance(user_stmt, parsing.Import): + scopes = [imports.ImportPath(user_stmt, is_like_search, + evaluate.follow_path)] else: - stmt.start_pos = position - stmt.parent = scope - scopes = evaluate.follow_statement(stmt) + # just parse one statement, take it and evaluate it + r = parsing.PyFuzzyParser(path, source_path) + try: + stmt = r.top.statements[0] + except IndexError: + if is_like_search: + path_tuple = path, dot, like + else: + path_tuple = () + raise NotFoundError(scope, path_tuple) + else: + stmt.start_pos = position + stmt.parent = scope + scopes = evaluate.follow_statement(stmt) if is_like_search: return scopes, path, dot, like diff --git a/modules.py b/modules.py index 252f8a46..b79b0df2 100644 --- a/modules.py +++ b/modules.py @@ -144,67 +144,3 @@ class ModuleWithCursor(Module): return self._line_cache[line - 1] except IndexError: raise StopIteration() - - -def find_module(current_module, point_path): - """ - Find a module with a path (of the module, like usb.backend.libusb10). - - Relative imports: http://www.python.org/dev/peps/pep-0328 - are only used like this (py3000): from .module import name. - - :param current_ns_path: A path to the current namespace. - :param point_path: A name from the parser. - :return: The rest of the path, and the module top scope. - """ - def follow_str(ns, string): - debug.dbg('follow_module', ns, string) - if ns: - path = [ns[1]] - else: - path = None - debug.dbg('search_module', string, path, - current_module.path) - try: - i = imp.find_module(string, path) - except ImportError: - # find builtins (ommit path): - i = imp.find_module(string, builtin.module_find_path) - return i - - # TODO handle relative paths - they are included in the import object - current_namespace = None - builtin.module_find_path.insert(0, os.path.dirname(current_module.path)) - # now execute those paths - rest = [] - for i, s in enumerate(point_path): - try: - current_namespace = follow_str(current_namespace, s) - except ImportError: - if current_namespace: - rest = point_path[i:] - else: - raise ModuleNotFound( - 'The module you searched has not been found') - - builtin.module_find_path.pop(0) - path = current_namespace[1] - is_package_directory = current_namespace[2][2] == imp.PKG_DIRECTORY - - f = None - if is_package_directory or current_namespace[0]: - # is a directory module - if is_package_directory: - path += '/__init__.py' - with open(path) as f: - source = f.read() - else: - source = current_namespace[0].read() - if path.endswith('.py'): - f = Module(path, source) - else: - f = builtin.Parser(path=path) - else: - f = builtin.Parser(name=path) - - return f.parser.top, rest diff --git a/parsing.py b/parsing.py index a138456a..d0f53f61 100644 --- a/parsing.py +++ b/parsing.py @@ -458,9 +458,11 @@ class Import(Simple): :type from_ns: Name :param star: If a star is used -> from time import *. :type star: bool + :param defunct: An Import is valid or not. + :type defunct: bool """ - def __init__(self, start_pos, end_pos, namespace, alias='', \ - from_ns='', star=False, relative_count=None): + def __init__(self, start_pos, end_pos, namespace, alias='', from_ns='', \ + star=False, relative_count=None, defunct=False): super(Import, self).__init__(start_pos, end_pos) self.namespace = namespace @@ -477,6 +479,7 @@ class Import(Simple): self.star = star self.relative_count = relative_count + self.defunct = defunct def get_code(self): if self.alias: @@ -491,6 +494,8 @@ class Import(Simple): return "import " + ns_str + '\n' def get_defined_names(self): + if self.defunct: + return [] if self.star: return [self] return [self.alias] if self.alias else [self.namespace] @@ -930,6 +935,7 @@ class PyFuzzyParser(object): """ def __init__(self, code, module_path=None, user_position=(None,None)): self.user_position = user_position + self.user_stmt = None self.code = code + '\n' # end with \n, because the parser needs it # initialize global Scope @@ -960,6 +966,20 @@ class PyFuzzyParser(object): return (self._line_of_tokenize_restart + self._tokenize_end_pos[0], self._tokenize_end_pos[1]) + def check_user_stmt(self, i): + # the position is right + if i.start_pos < self.user_position <= i.end_pos: + if self.user_stmt is not None: + # if there is already a user position (another import, because + # imports are splitted) the names are checked. + for n in i.get_defined_names(): + if n.start_pos < self.user_position <= n.end_pos: + self.user_stmt = i + else: + self.user_stmt = i + print 'up', self.user_stmt + + def _parsedotname(self, pre_used_token=None): """ The dot name parser parses a name, variable or function and returns @@ -1011,6 +1031,7 @@ class PyFuzzyParser(object): continue_kw = [",", ";", "\n", ')'] \ + list(set(keyword.kwlist) - set(['as'])) while True: + defunct = False token_type, tok = self.next() if brackets and tok == '\n': self.next() @@ -1019,11 +1040,11 @@ class PyFuzzyParser(object): self.next() i, token_type, tok = self._parsedotname(self.current) if not i: - break + defunct = True name2 = None if tok == 'as': name2, token_type, tok = self._parsedotname() - imports.append((i, name2)) + imports.append((i, name2, defunct)) while tok not in continue_kw: token_type, tok = self.next() if not (tok == "," or brackets and tok == '\n'): @@ -1316,12 +1337,19 @@ class PyFuzzyParser(object): # import stuff elif tok == 'import': imports = self._parseimportlist() - for m, alias in imports: - i = Import(first_pos, self.end_pos, m, alias) + for m, alias, defunct in imports: + i = Import(first_pos, self.end_pos, m, alias, + defunct=defunct) + self.check_user_stmt(i) + self.user_stmt = i self.scope.add_import(i) debug.dbg("new import: %s" % (i), self.current) + if not imports: + i = Import(first_pos, self.end_pos, None, defunct=True) + self.check_user_stmt(i) self.freshscope = False elif tok == 'from': + defunct = False # take care for relative imports relative_count = 0 while 1: @@ -1334,16 +1362,21 @@ class PyFuzzyParser(object): if not mod or tok != "import": debug.warning("from: syntax error@%s" % self.start_pos[0]) - continue + defunct = True names = self._parseimportlist() - for name, alias in names: + for name, alias, defunct2 in names: star = name.names[0] == '*' if star: name = None - i = Import(first_pos, self.end_pos, name, - alias, mod, star, relative_count) + i = Import(first_pos, self.end_pos, name, alias, mod, + star, relative_count, defunct=defunct or defunct2) + self.check_user_stmt(i) self.scope.add_import(i) debug.dbg("new from: %s" % (i)) + if not names: + i = Import(first_pos, self.end_pos, mod, defunct=True, + relative_count=relative_count) + self.check_user_stmt(i) self.freshscope = False #loops elif tok == 'for': diff --git a/test/completion/imports.py b/test/completion/imports.py index 5f7bde35..bba14755 100644 --- a/test/completion/imports.py +++ b/test/completion/imports.py @@ -52,3 +52,13 @@ def func_with_import(): #? ['sleep'] func_with_import().sleep + +# ----------------- +# completions within imports +# ----------------- + +#? ['sqlite3'] +import sqlite + +#? ['time'] +from datetime import diff --git a/test/run.py b/test/run.py index c792e456..d161082b 100755 --- a/test/run.py +++ b/test/run.py @@ -26,7 +26,7 @@ def run_completion_test(correct, source, line_nr, line): # lines start with 1 and column is just the last (makes no # difference for testing) try: - completions = functions.complete(source, line_nr, 999, + completions = functions.complete(source, line_nr, len(line), completion_test_dir) except (Exception, functions.evaluate.MultiLevelAttributeError): print('test @%s: %s' % (line_nr - 1, line)) @@ -51,7 +51,7 @@ def run_definition_test(correct, source, line_nr, line, correct_start): return set(functions.get_definitions(source, line_nr, indent, completion_test_dir)) try: - result = defs(line_nr, 999) + result = defs(line_nr, len(line)) except (Exception, functions.evaluate.MultiLevelAttributeError): print('test @%s: %s' % (line_nr - 1, line)) print(traceback.format_exc())