From d481a7aae49649d37d00fb2c85f161810f5d0fbc Mon Sep 17 00:00:00 2001 From: Aldo Stracquadanio Date: Fri, 22 Mar 2013 13:39:07 +0000 Subject: [PATCH 1/7] Created find_module helper to handle compatibility with python 3.3 Moved package checking logic in follow_str function Created find_module compatibility helper method Conditional implementation of load_module for python 3.3 --- jedi/_compatibility.py | 43 +++++++++++++++++++++++++++++++++++++++++- jedi/imports.py | 20 ++++++++++++++------ 2 files changed, 56 insertions(+), 7 deletions(-) diff --git a/jedi/_compatibility.py b/jedi/_compatibility.py index 9a704304..454511dd 100644 --- a/jedi/_compatibility.py +++ b/jedi/_compatibility.py @@ -7,11 +7,52 @@ Most of the code here is necessary to support Python 2.5. Once this dependency will be dropped, we'll get rid of most code. """ import sys +import imp +import os +import io +try: + import importlib +except: + pass is_py3k = sys.hexversion >= 0x03000000 - +is_py33 = sys.hexversion >= 0x03030000 is_py25 = sys.hexversion < 0x02060000 +if is_py33: + def find_module(string, path=None): + importing = None + if path is not None: + importing = importlib.find_loader(string, path) + else: + importing = importlib.find_loader(string) + + returning = (None, None, None) + try: + filename = importing.get_filename(string) + if filename and os.path.exists(filename): + returning = (open(filename, 'U'), filename, False) + else: + returning = (None, filename, False) + except AttributeError: + if importing is None: + returning = (None, None, False) + else: + returning = (None, importing.load_module(string).__name__, False) + + return returning +else: + def find_module(string, path=None): + importing = None + if path is None: + importing = imp.find_module(string) + else: + importing = imp.find_module(string, path) + + returning = (importing[0], importing[1], importing[2][2] == imp.PKG_DIRECTORY) + + return returning + # next was defined in python 2.6, in python 3 obj.next won't be possible # anymore try: diff --git a/jedi/imports.py b/jedi/imports.py index 72a3d0f6..a006f3e7 100644 --- a/jedi/imports.py +++ b/jedi/imports.py @@ -21,6 +21,7 @@ import imp import sys import itertools +from jedi._compatibility import find_module from jedi import modules from jedi import debug from jedi import parsing_representation as pr @@ -238,20 +239,22 @@ class ImportPath(pr.Base): global imports_processed imports_processed += 1 + importing = None if path is not None: - return imp.find_module(string, [path]) + importing = find_module(string, [path]) else: debug.dbg('search_module', string, self.file_path) # Override the sys.path. It works only good that way. # Injecting the path directly into `find_module` did not work. sys.path, temp = sys_path_mod, sys.path try: - i = imp.find_module(string) + importing = find_module(string) except ImportError: sys.path = temp raise sys.path = temp - return i + + return importing if self.file_path: sys_path_mod = list(self.sys_path_with_modifications()) @@ -259,6 +262,9 @@ class ImportPath(pr.Base): else: sys_path_mod = list(modules.get_sys_path()) + def module_not_found(): + raise ModuleNotFound('The module you searched has not been found') + current_namespace = (None, None, None) # now execute those paths rest = [] @@ -277,12 +283,14 @@ class ImportPath(pr.Base): if current_namespace[1]: rest = self.import_path[i:] else: - raise ModuleNotFound( - 'The module you searched has not been found') + module_not_found() + + if current_namespace == (None, None, False): + module_not_found() sys_path_mod.pop(0) # TODO why is this here? path = current_namespace[1] - is_package_directory = current_namespace[2][2] == imp.PKG_DIRECTORY + is_package_directory = current_namespace[2] f = None if is_package_directory or current_namespace[0]: From de849fbe8a1e08650f9556404a1959249375d8d6 Mon Sep 17 00:00:00 2001 From: Aldo Stracquadanio Date: Fri, 22 Mar 2013 19:11:44 +0000 Subject: [PATCH 2/7] Fixed follow_definition test --- jedi/_compatibility.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/jedi/_compatibility.py b/jedi/_compatibility.py index 454511dd..dff2e42d 100644 --- a/jedi/_compatibility.py +++ b/jedi/_compatibility.py @@ -24,6 +24,8 @@ if is_py33: importing = None if path is not None: importing = importlib.find_loader(string, path) + if importing is None: + raise ImportError else: importing = importlib.find_loader(string) @@ -39,7 +41,7 @@ if is_py33: returning = (None, None, False) else: returning = (None, importing.load_module(string).__name__, False) - + return returning else: def find_module(string, path=None): From be8ef33b695ed3a37972d9caff9a8e1c0c00bb61 Mon Sep 17 00:00:00 2001 From: Aldo Stracquadanio Date: Sat, 23 Mar 2013 23:07:17 +0000 Subject: [PATCH 3/7] Fixed caching issues Fixed exception raised during cache reading Switched cache hashing to hashlib In python 3.3 the hash function is returning different hashes during different executions of the application. --- jedi/cache.py | 3 ++- jedi/parsing_representation.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/jedi/cache.py b/jedi/cache.py index 20805845..2b671700 100644 --- a/jedi/cache.py +++ b/jedi/cache.py @@ -21,6 +21,7 @@ from __future__ import with_statement import time import os import sys +import hashlib try: import cPickle as pickle except: @@ -312,7 +313,7 @@ class _ModulePickling(object): shutil.rmtree(self._cache_directory()) def _get_hashed_path(self, path): - return self._get_path('%s.pkl' % hash(path)) + return self._get_path('%s.pkl' % hashlib.md5(path.encode("utf-8")).hexdigest()) def _get_path(self, file): dir = self._cache_directory() diff --git a/jedi/parsing_representation.py b/jedi/parsing_representation.py index 65210124..d1ea83ff 100644 --- a/jedi/parsing_representation.py +++ b/jedi/parsing_representation.py @@ -560,7 +560,7 @@ class Flow(Scope): @parent.setter def parent(self, value): self._parent = value - if self.next: + if hasattr(self, "next") and self.next: self.next.parent = value def get_code(self, first_indent=False, indention=' '): From 3ef564847d460c782de2cf2cccd16e9a8eba5567 Mon Sep 17 00:00:00 2001 From: Aldo Stracquadanio Date: Sat, 23 Mar 2013 23:16:06 +0000 Subject: [PATCH 4/7] Simplified code for readability Splitted import compatibility function definition for better readability Simplified code for python 3.3 load_module implementation --- jedi/_compatibility.py | 73 ++++++++++++++++++++++++------------------ 1 file changed, 42 insertions(+), 31 deletions(-) diff --git a/jedi/_compatibility.py b/jedi/_compatibility.py index dff2e42d..a3489869 100644 --- a/jedi/_compatibility.py +++ b/jedi/_compatibility.py @@ -19,41 +19,52 @@ is_py3k = sys.hexversion >= 0x03000000 is_py33 = sys.hexversion >= 0x03030000 is_py25 = sys.hexversion < 0x02060000 -if is_py33: - def find_module(string, path=None): - importing = None - if path is not None: - importing = importlib.find_loader(string, path) - if importing is None: - raise ImportError +def find_module_py33(string, path=None): + returning = (None, None, None) + importing = None + if path is not None: + importing = importlib.find_loader(string, path) + else: + importing = importlib.find_loader(string) + + if importing is None: + raise ImportError + + try: + filename = importing.get_filename(string) + if filename and os.path.exists(filename): + returning = (open(filename, 'U'), filename, False) else: - importing = importlib.find_loader(string) + returning = (None, filename, False) + except AttributeError: + returning = (None, importing.load_module(string).__name__, False) - returning = (None, None, None) - try: - filename = importing.get_filename(string) - if filename and os.path.exists(filename): - returning = (open(filename, 'U'), filename, False) - else: - returning = (None, filename, False) - except AttributeError: - if importing is None: - returning = (None, None, False) - else: - returning = (None, importing.load_module(string).__name__, False) - - return returning -else: - def find_module(string, path=None): - importing = None - if path is None: - importing = imp.find_module(string) - else: - importing = imp.find_module(string, path) + return returning - returning = (importing[0], importing[1], importing[2][2] == imp.PKG_DIRECTORY) +def find_module_pre_py33(string, path=None): + importing = None + if path is None: + importing = imp.find_module(string) + else: + importing = imp.find_module(string, path) - return returning + return (importing[0], importing[1], importing[2][2] == imp.PKG_DIRECTORY) + +def find_module(string, path=None): + """Provides information about a module. + + This function isolates the differences in importing libraries introduced with + python 3.3 on; it gets a module name and optionally a path. It will return a + tuple containin an open file for the module (if not builtin), the filename + or the name of the module if it is a builtin one and a boolean indicating + ig the module is contained in a package.""" + returning = (None, None, None) + if is_py33: + returning = find_module_py33(string, path) + else: + returning = find_module_pre_py33(string, path) + + return returning # next was defined in python 2.6, in python 3 obj.next won't be possible # anymore From 07ec134bc92736aafdf8b1203297431ca54eebd2 Mon Sep 17 00:00:00 2001 From: Aldo Stracquadanio Date: Sun, 24 Mar 2013 10:14:14 +0000 Subject: [PATCH 5/7] Adding python 3.3 to test environment, mani fixes Added python 3.3 to test-suite Removed unused import Removed unused import Migrated to EAFP for attribute checking Bumped version of ModulePickling for migration to hashlib Added py33 environment to tox Fixed issue with package importing on python 3.3 --- .travis.yml | 1 + jedi/_compatibility.py | 12 +++++++----- jedi/cache.py | 2 +- jedi/imports.py | 6 ++---- jedi/parsing_representation.py | 4 +++- tox.ini | 2 +- 6 files changed, 15 insertions(+), 12 deletions(-) diff --git a/.travis.yml b/.travis.yml index e1529542..a85a4d06 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,6 +4,7 @@ env: - TOXENV=py26 - TOXENV=py27 - TOXENV=py32 + - TOXENV=py33 install: - pip install --quiet --use-mirrors tox script: diff --git a/jedi/_compatibility.py b/jedi/_compatibility.py index a3489869..331c1f11 100644 --- a/jedi/_compatibility.py +++ b/jedi/_compatibility.py @@ -9,7 +9,6 @@ will be dropped, we'll get rid of most code. import sys import imp import os -import io try: import importlib except: @@ -31,11 +30,14 @@ def find_module_py33(string, path=None): raise ImportError try: - filename = importing.get_filename(string) - if filename and os.path.exists(filename): - returning = (open(filename, 'U'), filename, False) + if (importing.is_package(string)): + returning = (None, os.path.dirname(importing.path), True) else: - returning = (None, filename, False) + filename = importing.get_filename(string) + if filename and os.path.exists(filename): + returning = (open(filename, 'U'), filename, False) + else: + returning = (None, filename, False) except AttributeError: returning = (None, importing.load_module(string).__name__, False) diff --git a/jedi/cache.py b/jedi/cache.py index 2b671700..5d76f24e 100644 --- a/jedi/cache.py +++ b/jedi/cache.py @@ -222,7 +222,7 @@ def save_module(path, name, parser, pickling=True): class _ModulePickling(object): - version = 1 + version = 2 """ Version number (integer) for file system cache. diff --git a/jedi/imports.py b/jedi/imports.py index a006f3e7..db72925f 100644 --- a/jedi/imports.py +++ b/jedi/imports.py @@ -5,9 +5,8 @@ any actual importing done. This module is about finding modules in the filesystem. This can be quite tricky sometimes, because Python imports are not always that simple. -Currently the import process uses ``imp`` to find modules. In the future, it's -a goal to use ``importlib`` for this purpose. There's a `pull request -`_ for that. +This module uses imp for python up to 3.2 and importlib for python 3.3 on; the +correct implementation is delegated to _compatibility. This module also supports import autocompletion, which means to complete statements like ``from datetim`` (curser at the end would return ``datetime``). @@ -17,7 +16,6 @@ from __future__ import with_statement import os import pkgutil -import imp import sys import itertools diff --git a/jedi/parsing_representation.py b/jedi/parsing_representation.py index d1ea83ff..311f38ab 100644 --- a/jedi/parsing_representation.py +++ b/jedi/parsing_representation.py @@ -560,8 +560,10 @@ class Flow(Scope): @parent.setter def parent(self, value): self._parent = value - if hasattr(self, "next") and self.next: + try: self.next.parent = value + except AttributeError: + return def get_code(self, first_indent=False, indention=' '): stmts = [] diff --git a/tox.ini b/tox.ini index 6dc16930..8fa50614 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py25, py26, py27, py32 +envlist = py25, py26, py27, py32, py33 [testenv] deps = pytest From 0b67a08e48ee4c1430a3557f9bbccc150547aacc Mon Sep 17 00:00:00 2001 From: Aldo Stracquadanio Date: Mon, 25 Mar 2013 19:30:58 +0000 Subject: [PATCH 6/7] Using PathFinder rather than find_loader to correctly handle paths Using PathFinder rather than find_loader to correctly handle from ... import ... Moved away from find_loader in favour of PathFinder also when using sys.path --- jedi/_compatibility.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/jedi/_compatibility.py b/jedi/_compatibility.py index 331c1f11..e15ba463 100644 --- a/jedi/_compatibility.py +++ b/jedi/_compatibility.py @@ -22,9 +22,11 @@ def find_module_py33(string, path=None): returning = (None, None, None) importing = None if path is not None: - importing = importlib.find_loader(string, path) + importing = importlib.machinery.PathFinder.find_module(string, path) else: - importing = importlib.find_loader(string) + importing = importlib.machinery.PathFinder.find_module(string, sys.path) + if importing is None: + importing = importlib.find_loader(string) if importing is None: raise ImportError From 124595de6e503d5e840a43d893c1018190fd8413 Mon Sep 17 00:00:00 2001 From: Aldo Stracquadanio Date: Wed, 27 Mar 2013 00:19:39 +0000 Subject: [PATCH 7/7] Making it nicer Fixed typo in docstring and added some comments in find_module_py33 Removed a test that is not compatible with python 3.3 Better variable names in find_module implementation(s) Removed variable assignation in favor of direct return statement --- jedi/_compatibility.py | 48 +++++++++++++++++++++--------------------- test/completion/std.py | 9 -------- 2 files changed, 24 insertions(+), 33 deletions(-) diff --git a/jedi/_compatibility.py b/jedi/_compatibility.py index e15ba463..29aef154 100644 --- a/jedi/_compatibility.py +++ b/jedi/_compatibility.py @@ -19,40 +19,43 @@ is_py33 = sys.hexversion >= 0x03030000 is_py25 = sys.hexversion < 0x02060000 def find_module_py33(string, path=None): - returning = (None, None, None) - importing = None + mod_info = (None, None, None) + loader = None if path is not None: - importing = importlib.machinery.PathFinder.find_module(string, path) + # Check for the module in the specidied path + loader = importlib.machinery.PathFinder.find_module(string, path) else: - importing = importlib.machinery.PathFinder.find_module(string, sys.path) - if importing is None: - importing = importlib.find_loader(string) + # Check for the module in sys.path + loader = importlib.machinery.PathFinder.find_module(string, sys.path) + if loader is None: + # Fallback to find builtins + loader = importlib.find_loader(string) - if importing is None: + if loader is None: raise ImportError try: - if (importing.is_package(string)): - returning = (None, os.path.dirname(importing.path), True) + if (loader.is_package(string)): + mod_info = (None, os.path.dirname(loader.path), True) else: - filename = importing.get_filename(string) + filename = loader.get_filename(string) if filename and os.path.exists(filename): - returning = (open(filename, 'U'), filename, False) + mod_info = (open(filename, 'U'), filename, False) else: - returning = (None, filename, False) + mod_info = (None, filename, False) except AttributeError: - returning = (None, importing.load_module(string).__name__, False) + mod_info = (None, loader.load_module(string).__name__, False) - return returning + return mod_info def find_module_pre_py33(string, path=None): - importing = None + mod_info = None if path is None: - importing = imp.find_module(string) + mod_info = imp.find_module(string) else: - importing = imp.find_module(string, path) + mod_info = imp.find_module(string, path) - return (importing[0], importing[1], importing[2][2] == imp.PKG_DIRECTORY) + return (mod_info[0], mod_info[1], mod_info[2][2] == imp.PKG_DIRECTORY) def find_module(string, path=None): """Provides information about a module. @@ -61,14 +64,11 @@ def find_module(string, path=None): python 3.3 on; it gets a module name and optionally a path. It will return a tuple containin an open file for the module (if not builtin), the filename or the name of the module if it is a builtin one and a boolean indicating - ig the module is contained in a package.""" - returning = (None, None, None) + if the module is contained in a package.""" if is_py33: - returning = find_module_py33(string, path) + return find_module_py33(string, path) else: - returning = find_module_pre_py33(string, path) - - return returning + return find_module_pre_py33(string, path) # next was defined in python 2.6, in python 3 obj.next won't be possible # anymore diff --git a/test/completion/std.py b/test/completion/std.py index 0eec6367..c4ed8c5a 100644 --- a/test/completion/std.py +++ b/test/completion/std.py @@ -89,12 +89,3 @@ def huhu(db): """ #? sqlite3.Connection() db - -# ----------------- -# various regression tests -# ----------------- - -#62 -import threading -#? ['_Verbose', '_VERBOSE'] -threading._Verbose