developed star import caching. useful for numpy/pylab/scipy... fixes jedi-vim issue 23

jedi/builtin.py

@@ -12,6 +12,7 @@ import inspect
 
 import debug
 import parsing
+import imports
 
 
 def get_sys_path():
@@ -50,6 +51,10 @@ class CachedModule(object):
                 if not self.path or os.path.getmtime(self.path) <= timestamp:
                     self._parser = parser
                 else:
+                    # In case there is already a module cached and this module
+                    # has to be reparsed, we also need to invalidate the import
+                    # caches.
+                    imports.invalidate_star_import_cache(self._parser.module)
                     raise KeyError()
             except KeyError:
                 self._load_module()

jedi/imports.py

@@ -5,6 +5,7 @@ import pkgutil
 import imp
 import sys
 import weakref
+import time
 
 import builtin
 import modules
@@ -12,10 +13,13 @@ import debug
 import parsing
 import evaluate
 import itertools
+import settings
 
 # for debugging purposes only
 imports_processed = 0
 
+star_import_cache = {}
+
 
 class ModuleNotFound(Exception):
     pass
@@ -264,12 +268,42 @@ def strip_imports(scopes):
     return result
 
 
-def star_import_cache(func):
+def cache_star_import(func):
     def wrapper(scope, *args, **kwargs):
-        return func(scope, *args, **kwargs)
+        try:
+            mods = star_import_cache[scope]
+            if mods[0] + settings.star_import_cache_validity > time.time():
+                return mods[1]
+        except KeyError:
+            pass
+        # cache is too old and therefore invalid or not available
+        invalidate_star_import_cache(scope)
+        mods = func(scope, *args, **kwargs)
+        star_import_cache[scope] = time.time(), mods
+
+        return mods
     return wrapper
 
-@star_import_cache
+
+def invalidate_star_import_cache(module, only_main=False):
+    """ Important if some new modules are being reparsed """
+    try:
+        mods = star_import_cache[module]
+
+        for t, m in mods:
+            invalidate_star_import_cache(m, only_main=True)
+
+        del star_import_cache[module]
+    except KeyError:
+        pass
+
+    if not only_main:
+        for key, (t, mods) in star_import_cache.items():
+            if module in mods:
+                invalidate_star_import_cache(key)
+
+
+@cache_star_import
 def remove_star_imports(scope, ignored_modules=[]):
     """
     Check a module for star imports:
@@ -285,6 +319,4 @@ def remove_star_imports(scope, ignored_modules=[]):
     modules += new
 
     # Filter duplicate modules.
-    if len(modules) > 10:
-        print scope, len(modules)
     return set(modules)

jedi/settings.py

@@ -75,3 +75,13 @@ scale_get_in_function_call = 0.1
 
 # Size of the current code part, which is used to speed up parsing.
 part_line_length = 20
+
+# ----------------
+# star import caching
+# ----------------
+
+# In huge packages like numpy, checking all star imports on every completion
+# might be slow, therefore we do a star import caching, that lasts a certain
+# time span (in seconds).
+
+star_import_cache_validity = 60.0

test/regression.py

@@ -249,7 +249,7 @@ class TestSpeed(Base):
                 for i in range(number):
                     func(self)
                 single_time = (time.time() - first) / number
-                print(func, single_time)
+                print('speed', func, single_time)
                 assert single_time < time_per_run
             return wrapper
         return decorated
@@ -259,15 +259,12 @@ class TestSpeed(Base):
         s = "from posixpath import join; join('', '')."
         assert len(self.complete(s)) > 10  # is a str completion
 
-    @_check_speed(0.2, number=1)
+    @_check_speed(0.1)
     def test_scipy_speed(self):
         s = 'import scipy.weave; scipy.weave.inline('
-        #api.set_debug_function(api.debug.print_to_stdout)
         script = api.Script(s, 1, len(s), '')
         script.get_in_function_call()
-        # self.get_in_function_call(s)
+        #print(api.imports.imports_processed)
-        #api.set_debug_function(None)
-        print(api.imports.imports_processed)
 
 
 if __name__ == '__main__':