basic imports rewriting, which has still it's rough edges

evaluate.py

@@ -30,6 +30,7 @@ import parsing
 import modules
 import debug
 import builtin
+import imports
 import helpers
 
 memoize_caches = []
@@ -421,7 +422,7 @@ class Execution(Executable):
 
         debug.dbg('exec result: %s in %s' % (stmts, self))
 
-        return strip_imports(stmts)
+        return imports.strip_imports(stmts)
 
     def _get_function_returns(self, evaluate_generator):
         func = self.base
@@ -815,7 +816,7 @@ def get_names_for_scope(scope, position=None, star_search=True):
 
     # Add star imports.
     if star_search:
-        for s in remove_star_imports(start_scope.get_parent_until()):
+        for s in imports.remove_star_imports(start_scope.get_parent_until()):
             for g in get_names_for_scope(s, star_search=False):
                 yield g
 
@@ -938,23 +939,6 @@ def get_scopes_for_name(scope, name_str, position=None, search_global=False):
     return remove_statements(filter_name(scope_generator))
 
 
-def strip_imports(scopes):
-    """
-    Here we strip the imports - they don't get resolved necessarily.
-    Really used anymore?
-    """
-    result = []
-    for s in scopes:
-        if isinstance(s, parsing.Import):
-            try:
-                result += follow_import(s)
-            except modules.ModuleNotFound:
-                debug.warning('Module not found: ' + str(s))
-        else:
-            result.append(s)
-    return result
-
-
 def assign_tuples(tup, results, seek_name):
     """
     This is a normal assignment checker. In python functions and other things
@@ -1099,7 +1083,7 @@ def follow_call(call):
             # This is the first global lookup.
             scopes = get_scopes_for_name(scope, current, position=position,
                                             search_global=True)
-        result = strip_imports(scopes)
+        result = imports.strip_imports(scopes)
 
         if result != scopes:
             # Reset the position, when imports where stripped.
@@ -1162,49 +1146,8 @@ def follow_path(path, scope, position=None):
         else:
             # TODO Check magic class methods and return them also.
             # This is the typical lookup while chaining things.
-            result = strip_imports(get_scopes_for_name(scope, current,
+            result = imports.strip_imports(get_scopes_for_name(scope, current,
                                                         position=position))
     return follow_paths(path, result, position=position)
 
 
-def follow_import(_import):
-    """
-    follows a module name and returns the parser.
-    :param _import: The import statement.
-    :type _import: parsing.Import
-    """
-    # Set path together.
-    ns_list = []
-    if _import.from_ns:
-        ns_list += _import.from_ns.names
-    if _import.namespace:
-        ns_list += _import.namespace.names
-
-    loaded_in = _import.get_parent_until()
-
-    scope, rest = modules.find_module(loaded_in, ns_list)
-    if rest:
-        scopes = follow_path(iter(rest), scope)
-    else:
-        scopes = [scope]
-
-    new = []
-    for scope in scopes:
-        new += remove_star_imports(scope)
-    scopes += new
-
-    debug.dbg('after import', scopes, rest)
-    return scopes
-
-
-def remove_star_imports(scope):
-    """
-    """
-    modules = strip_imports(i for i in scope.get_imports() if i.star)
-    new = []
-    for m in modules:
-        new += remove_star_imports(m)
-    modules += new
-
-    # Filter duplicate modules.
-    return set(modules)

functions.py

@@ -5,6 +5,7 @@ import parsing
 import evaluate
 import modules
 import debug
+import imports
 
 __all__ = ['complete', 'get_completion_parts', 'get_definitions',
             'set_debug_function']
@@ -183,20 +184,25 @@ def prepare_goto(source, position, source_path, is_like_search):
 
     debug.dbg('start: %s in %s' % (path, scope))
 
-    # just parse one statement, take it and evaluate it
+    user_stmt = f.parser.user_stmt
-    r = parsing.PyFuzzyParser(path, source_path)
+    if isinstance(user_stmt, parsing.Import):
-    try:
+        scopes = [imports.ImportPath(user_stmt, is_like_search,
-        stmt = r.top.statements[0]
+                                                    evaluate.follow_path)]
-    except IndexError:
-        if is_like_search:
-            path_tuple = path, dot, like
-        else:
-            path_tuple = ()
-        raise NotFoundError(scope, path_tuple)
     else:
-        stmt.start_pos = position
+        # just parse one statement, take it and evaluate it
-        stmt.parent = scope
+        r = parsing.PyFuzzyParser(path, source_path)
-        scopes = evaluate.follow_statement(stmt)
+        try:
+            stmt = r.top.statements[0]
+        except IndexError:
+            if is_like_search:
+                path_tuple = path, dot, like
+            else:
+                path_tuple = ()
+            raise NotFoundError(scope, path_tuple)
+        else:
+            stmt.start_pos = position
+            stmt.parent = scope
+            scopes = evaluate.follow_statement(stmt)
 
     if is_like_search:
         return scopes, path, dot, like

modules.py

@@ -144,67 +144,3 @@ class ModuleWithCursor(Module):
             return self._line_cache[line - 1]
         except IndexError:
             raise StopIteration()
-
-
-def find_module(current_module, point_path):
-    """
-    Find a module with a path (of the module, like usb.backend.libusb10).
-
-    Relative imports: http://www.python.org/dev/peps/pep-0328
-    are only used like this (py3000): from .module import name.
-
-    :param current_ns_path: A path to the current namespace.
-    :param point_path: A name from the parser.
-    :return: The rest of the path, and the module top scope.
-    """
-    def follow_str(ns, string):
-        debug.dbg('follow_module', ns, string)
-        if ns:
-            path = [ns[1]]
-        else:
-            path = None
-            debug.dbg('search_module', string, path,
-                                            current_module.path)
-        try:
-            i = imp.find_module(string, path)
-        except ImportError:
-            # find builtins (ommit path):
-            i = imp.find_module(string, builtin.module_find_path)
-        return i
-
-    # TODO handle relative paths - they are included in the import object
-    current_namespace = None
-    builtin.module_find_path.insert(0, os.path.dirname(current_module.path))
-    # now execute those paths
-    rest = []
-    for i, s in enumerate(point_path):
-        try:
-            current_namespace = follow_str(current_namespace, s)
-        except ImportError:
-            if current_namespace:
-                rest = point_path[i:]
-            else:
-                raise ModuleNotFound(
-                        'The module you searched has not been found')
-
-    builtin.module_find_path.pop(0)
-    path = current_namespace[1]
-    is_package_directory = current_namespace[2][2] == imp.PKG_DIRECTORY
-
-    f = None
-    if is_package_directory or current_namespace[0]:
-        # is a directory module
-        if is_package_directory:
-            path += '/__init__.py'
-            with open(path) as f:
-                source = f.read()
-        else:
-            source = current_namespace[0].read()
-        if path.endswith('.py'):
-            f = Module(path, source)
-        else:
-            f = builtin.Parser(path=path)
-    else:
-        f = builtin.Parser(name=path)
-
-    return f.parser.top, rest

parsing.py

@@ -458,9 +458,11 @@ class Import(Simple):
     :type from_ns: Name
     :param star: If a star is used -> from time import *.
     :type star: bool
+    :param defunct: An Import is valid or not.
+    :type defunct: bool
     """
-    def __init__(self, start_pos, end_pos, namespace, alias='', \
+    def __init__(self, start_pos, end_pos, namespace, alias='', from_ns='', \
-                    from_ns='', star=False, relative_count=None):
+                    star=False, relative_count=None, defunct=False):
         super(Import, self).__init__(start_pos, end_pos)
 
         self.namespace = namespace
@@ -477,6 +479,7 @@ class Import(Simple):
 
         self.star = star
         self.relative_count = relative_count
+        self.defunct = defunct
 
     def get_code(self):
         if self.alias:
@@ -491,6 +494,8 @@ class Import(Simple):
             return "import " + ns_str + '\n'
 
     def get_defined_names(self):
+        if self.defunct:
+            return []
         if self.star:
             return [self]
         return [self.alias] if self.alias else [self.namespace]
@@ -930,6 +935,7 @@ class PyFuzzyParser(object):
     """
     def __init__(self, code, module_path=None, user_position=(None,None)):
         self.user_position = user_position
+        self.user_stmt = None
         self.code = code + '\n'  # end with \n, because the parser needs it
 
         # initialize global Scope
@@ -960,6 +966,20 @@ class PyFuzzyParser(object):
         return (self._line_of_tokenize_restart + self._tokenize_end_pos[0],
                                                 self._tokenize_end_pos[1])
 
+    def check_user_stmt(self, i):
+        # the position is right
+        if i.start_pos < self.user_position <= i.end_pos:
+            if self.user_stmt is not None:
+                # if there is already a user position (another import, because
+                # imports are splitted) the names are checked.
+                for n in i.get_defined_names():
+                    if n.start_pos < self.user_position <= n.end_pos:
+                        self.user_stmt = i
+            else:
+                self.user_stmt = i
+            print 'up', self.user_stmt
+
+
     def _parsedotname(self, pre_used_token=None):
         """
         The dot name parser parses a name, variable or function and returns
@@ -1011,6 +1031,7 @@ class PyFuzzyParser(object):
         continue_kw = [",", ";", "\n", ')'] \
                         + list(set(keyword.kwlist) - set(['as']))
         while True:
+            defunct = False
             token_type, tok = self.next()
             if brackets and tok == '\n':
                 self.next()
@@ -1019,11 +1040,11 @@ class PyFuzzyParser(object):
                 self.next()
             i, token_type, tok = self._parsedotname(self.current)
             if not i:
-                break
+                defunct = True
             name2 = None
             if tok == 'as':
                 name2, token_type, tok = self._parsedotname()
-            imports.append((i, name2))
+            imports.append((i, name2, defunct))
             while tok not in continue_kw:
                 token_type, tok = self.next()
             if not (tok == "," or brackets and tok == '\n'):
@@ -1316,12 +1337,19 @@ class PyFuzzyParser(object):
                 # import stuff
                 elif tok == 'import':
                     imports = self._parseimportlist()
-                    for m, alias in imports:
+                    for m, alias, defunct in imports:
-                        i = Import(first_pos, self.end_pos, m, alias)
+                        i = Import(first_pos, self.end_pos, m, alias,
+                                                            defunct=defunct)
+                        self.check_user_stmt(i)
+                        self.user_stmt = i
                         self.scope.add_import(i)
                         debug.dbg("new import: %s" % (i), self.current)
+                    if not imports:
+                        i = Import(first_pos, self.end_pos, None, defunct=True)
+                        self.check_user_stmt(i)
                     self.freshscope = False
                 elif tok == 'from':
+                    defunct = False
                     # take care for relative imports
                     relative_count = 0
                     while 1:
@@ -1334,16 +1362,21 @@ class PyFuzzyParser(object):
                     if not mod or tok != "import":
                         debug.warning("from: syntax error@%s" %
                                                             self.start_pos[0])
-                        continue
+                        defunct = True
                     names = self._parseimportlist()
-                    for name, alias in names:
+                    for name, alias, defunct2 in names:
                         star = name.names[0] == '*'
                         if star:
                             name = None
-                        i = Import(first_pos, self.end_pos, name,
+                        i = Import(first_pos, self.end_pos, name, alias, mod,
-                                    alias, mod, star, relative_count)
+                            star, relative_count, defunct=defunct or defunct2)
+                        self.check_user_stmt(i)
                         self.scope.add_import(i)
                         debug.dbg("new from: %s" % (i))
+                    if not names:
+                        i = Import(first_pos, self.end_pos, mod, defunct=True,
+                                    relative_count=relative_count)
+                        self.check_user_stmt(i)
                     self.freshscope = False
                 #loops
                 elif tok == 'for':

test/completion/imports.py

@@ -52,3 +52,13 @@ def func_with_import():
 
 #? ['sleep']
 func_with_import().sleep
+
+# -----------------
+# completions within imports
+# -----------------
+
+#? ['sqlite3']
+import sqlite
+
+#? ['time']
+from datetime import 

test/run.py

@@ -26,7 +26,7 @@ def run_completion_test(correct, source, line_nr, line):
     # lines start with 1 and column is just the last (makes no
     # difference for testing)
     try:
-        completions = functions.complete(source, line_nr, 999,
+        completions = functions.complete(source, line_nr, len(line),
                                             completion_test_dir)
     except (Exception, functions.evaluate.MultiLevelAttributeError):
         print('test @%s: %s' % (line_nr - 1, line))
@@ -51,7 +51,7 @@ def run_definition_test(correct, source, line_nr, line, correct_start):
         return set(functions.get_definitions(source, line_nr, indent,
                                             completion_test_dir))
     try:
-        result = defs(line_nr, 999)
+        result = defs(line_nr, len(line))
     except (Exception, functions.evaluate.MultiLevelAttributeError):
         print('test @%s: %s' % (line_nr - 1, line))
         print(traceback.format_exc())