VimPlug/jedi-fork
1
0
Fork 0
forked from VimPlug/jedi
Code Pull Requests Activity

Merge branch 'dev' into tokenizer

Browse Source
This commit is contained in:
David Halter
2013-04-20 08:55:02 +04:30
parent 3b0f4b87cf 36614fdb27
commit fa53579950
25 changed files with 305 additions and 119 deletions
Download Patch File Download Diff File Expand all files Collapse all files
jedi/_compatibility.py
+59 -1
View File
@@ -7,11 +7,69 @@ Most of the code here is necessary to support Python 2.5. Once this dependency
will be dropped, we'll get rid of most code.
"""
import sys
import imp
import os
try:
import importlib
except:
pass
is_py3k = sys.hexversion >= 0x03000000
is_py33 = sys.hexversion >= 0x03030000
is_py25 = sys.hexversion < 0x02060000
def find_module_py33(string, path=None):
mod_info = (None, None, None)
loader = None
if path is not None:
# Check for the module in the specidied path
loader = importlib.machinery.PathFinder.find_module(string, path)
else:
# Check for the module in sys.path
loader = importlib.machinery.PathFinder.find_module(string, sys.path)
if loader is None:
# Fallback to find builtins
loader = importlib.find_loader(string)
if loader is None:
raise ImportError
try:
if (loader.is_package(string)):
mod_info = (None, os.path.dirname(loader.path), True)
else:
filename = loader.get_filename(string)
if filename and os.path.exists(filename):
mod_info = (open(filename, 'U'), filename, False)
else:
mod_info = (None, filename, False)
except AttributeError:
mod_info = (None, loader.load_module(string).__name__, False)
return mod_info
def find_module_pre_py33(string, path=None):
mod_info = None
if path is None:
mod_info = imp.find_module(string)
else:
mod_info = imp.find_module(string, path)
return (mod_info[0], mod_info[1], mod_info[2][2] == imp.PKG_DIRECTORY)
def find_module(string, path=None):
"""Provides information about a module.
This function isolates the differences in importing libraries introduced with
python 3.3 on; it gets a module name and optionally a path. It will return a
tuple containin an open file for the module (if not builtin), the filename
or the name of the module if it is a builtin one and a boolean indicating
if the module is contained in a package."""
if is_py33:
return find_module_py33(string, path)
else:
return find_module_pre_py33(string, path)
# next was defined in python 2.6, in python 3 obj.next won't be possible
# anymore
try:
jedi/api_classes.py
+4 -6
View File
@@ -3,6 +3,7 @@ The :mod:`api_classes` module contains the return classes of the API. These
classes are the much bigger part of the whole API, because they contain the
interesting information about completion and goto operations.
"""
from __future__ import with_statement
import re
import os
@@ -11,6 +12,7 @@ import functools
from jedi._compatibility import unicode, next
from jedi import settings
from jedi import common
from jedi import parsing_representation as pr
from jedi import cache
import keywords
@@ -144,10 +146,8 @@ class BaseDefinition(object):
if not isinstance(self.definition, keywords.Keyword):
par = self.definition
while par is not None:
try:
with common.ignored(AttributeError):
path.insert(0, par.name)
except AttributeError:
pass
par = par.parent
return path
@@ -295,10 +295,8 @@ class BaseDefinition(object):
if not path:
return None # for keywords the path is empty
try:
with common.ignored(KeyError):
path[0] = self._mapping[path[0]]
except KeyError:
pass
for key, repl in self._tuple_mapping.items():
if tuple(path[:len(key)]) == key:
path = [repl] + path[len(key):]
jedi/cache.py
+55 -16
View File
@@ -21,10 +21,16 @@ from __future__ import with_statement
import time
import os
import sys
import pickle
import hashlib
try:
import cPickle as pickle
except:
import pickle
import shutil
from jedi._compatibility import json
from jedi import settings
from jedi import common
from jedi import debug
# memoize caches will be deleted after every action
@@ -143,12 +149,10 @@ def cache_function_definition(stmt):
def cache_star_import(func):
def wrapper(scope, *args, **kwargs):
try:
with common.ignored(KeyError):
mods = star_import_cache[scope]
if mods[0] + settings.star_import_cache_validity > time.time():
return mods[1]
except KeyError:
pass
# cache is too old and therefore invalid or not available
invalidate_star_import_cache(scope)
mods = func(scope, *args, **kwargs)
@@ -160,15 +164,13 @@ def cache_star_import(func):
def invalidate_star_import_cache(module, only_main=False):
""" Important if some new modules are being reparsed """
try:
with common.ignored(KeyError):
t, mods = star_import_cache[module]
del star_import_cache[module]
for m in mods:
invalidate_star_import_cache(m, only_main=True)
except KeyError:
pass
if not only_main:
# We need a list here because otherwise the list is being changed
@@ -216,13 +218,36 @@ def save_module(path, name, parser, pickling=True):
class _ModulePickling(object):
version = 2
"""
Version number (integer) for file system cache.
Increment this number when there are any incompatible changes in
parser representation classes. For example, the following changes
are regarded as incompatible.
- Class name is changed.
- Class is moved to another module.
- Defined slot of the class is changed.
"""
def __init__(self):
self.__index = None
self.py_version = '%s.%s' % sys.version_info[:2]
self.py_tag = 'cpython-%s%s' % sys.version_info[:2]
"""
Short name for distinguish Python implementations and versions.
It's like `sys.implementation.cache_tag` but for Python < 3.3
we generate something similar. See:
http://docs.python.org/3/library/sys.html#sys.implementation
.. todo:: Detect interpreter (e.g., PyPy).
"""
def load_module(self, path, original_changed_time):
try:
pickle_changed_time = self._index[self.py_version][path]
pickle_changed_time = self._index[path]
except KeyError:
return None
if original_changed_time is not None \
@@ -240,10 +265,10 @@ class _ModulePickling(object):
def save_module(self, path, parser_cache_item):
self.__index = None
try:
files = self._index[self.py_version]
files = self._index
except KeyError:
files = {}
self._index[self.py_version] = files
self._index = files
with open(self._get_hashed_path(path), 'wb') as f:
pickle.dump(parser_cache_item, f, pickle.HIGHEST_PROTOCOL)
@@ -256,9 +281,16 @@ class _ModulePickling(object):
if self.__index is None:
try:
with open(self._get_path('index.json')) as f:
self.__index = json.load(f)
data = json.load(f)
except IOError:
self.__index = {}
else:
# 0 means version is not defined (= always delete cache):
if data.get('version', 0) != self.version:
self.delete_cache()
self.__index = {}
else:
self.__index = data['index']
return self.__index
def _remove_old_modules(self):
@@ -269,18 +301,25 @@ class _ModulePickling(object):
self._index # reload index
def _flush_index(self):
data = {'version': self.version, 'index': self._index}
with open(self._get_path('index.json'), 'w') as f:
json.dump(self._index, f)
json.dump(data, f)
self.__index = None
def delete_cache(self):
shutil.rmtree(self._cache_directory())
def _get_hashed_path(self, path):
return self._get_path('%s_%s.pkl' % (self.py_version, hash(path)))
return self._get_path('%s.pkl' % hashlib.md5(path.encode("utf-8")).hexdigest())
def _get_path(self, file):
dir = settings.cache_directory
dir = self._cache_directory()
if not os.path.exists(dir):
os.makedirs(dir)
return dir + os.path.sep + file
return os.path.join(dir, file)
def _cache_directory(self):
return os.path.join(settings.cache_directory, self.py_tag)
# is a singleton
jedi/common.py
+10
View File
@@ -163,3 +163,13 @@ def indent_block(text, indention=' '):
text = text[:-1]
lines = text.split('\n')
return '\n'.join(map(lambda s: indention + s, lines)) + temp
@contextlib.contextmanager
def ignored(*exceptions):
"""Context manager that ignores all of the specified exceptions. This will
be in the standard library starting with Python 3.4."""
try:
yield
except exceptions:
pass
jedi/dynamic.py
+2 -3
View File
@@ -59,6 +59,7 @@ from jedi import cache
from jedi import parsing_representation as pr
from jedi import modules
from jedi import settings
from jedi import common
from jedi import debug
from jedi import fast_parser
import api_classes
@@ -487,10 +488,8 @@ def related_name_add_import_modules(definitions, search_name):
for d in definitions:
if isinstance(d.parent, pr.Import):
s = imports.ImportPath(d.parent, direct_resolve=True)
try:
with common.ignored(IndexError):
new.add(s.follow(is_goto=True)[0])
except IndexError:
pass
return set(definitions) | new
jedi/evaluate.py
+6 -15
View File
@@ -68,6 +68,7 @@ backtracking algorithm.
.. todo:: nonlocal statement, needed or can be ignored? (py3k)
"""
from __future__ import with_statement
import sys
import itertools
@@ -429,11 +430,9 @@ def find_name(scope, name_str, position=None, search_global=False,
if isinstance(scope, (er.Instance, er.Class)) \
and hasattr(r, 'get_descriptor_return'):
# handle descriptors
try:
with common.ignored(KeyError):
res_new += r.get_descriptor_return(scope)
continue
except KeyError:
pass
res_new.append(r)
return res_new
@@ -462,19 +461,15 @@ def check_getattr(inst, name_str):
# str is important to lose the NamePart!
module = builtin.Builtin.scope
name = pr.Call(module, str(name_str), pr.Call.STRING, (0, 0), inst)
try:
with common.ignored(KeyError):
result = inst.execute_subscope_by_name('__getattr__', [name])
except KeyError:
pass
if not result:
# this is a little bit special. `__getattribute__` is executed
# before anything else. But: I know no use case, where this
# could be practical and the jedi would return wrong types. If
# you ever have something, let me know!
try:
with common.ignored(KeyError):
result = inst.execute_subscope_by_name('__getattribute__', [name])
except KeyError:
pass
return result
@@ -536,10 +531,8 @@ def assign_tuples(tup, results, seek_name):
debug.warning("invalid tuple lookup %s of result %s in %s"
% (tup, results, seek_name))
else:
try:
with common.ignored(IndexError):
types += func(index)
except IndexError:
pass
return types
result = []
@@ -648,11 +641,9 @@ def follow_call_list(call_list, follow_array=False):
call = next(calls_iterator)
except StopIteration:
break
try:
with common.ignored(AttributeError):
if str(call.name) == 'else':
break
except AttributeError:
pass
continue
result += follow_call(call)
elif call == '*':
jedi/evaluate_representation.py
+11 -9
View File
@@ -9,6 +9,8 @@ instantiated. This class represents these cases.
So, why is there also a ``Class`` class here? Well, there are decorators and
they change classes in Python 3.
"""
from __future__ import with_statement
import copy
import itertools
@@ -61,10 +63,8 @@ class Instance(use_metaclass(cache.CachedMetaClass, Executable)):
else:
# need to execute the __init__ function, because the dynamic param
# searching needs it.
try:
with common.ignored(KeyError):
self.execute_subscope_by_name('__init__', self.var_args)
except KeyError:
pass
# Generated instances are classes that are just generated by self
# (No var_args) used.
self.is_generated = False
@@ -769,9 +769,13 @@ class Generator(use_metaclass(cache.CachedMetaClass, pr.Base)):
debug.warning('Tried to get array access on a generator', self)
return []
@property
def parent(self):
return self.func.parent
def __getattr__(self, name):
if name not in ['start_pos', 'end_pos', 'parent', 'get_imports',
'asserts', 'doc', 'docstr', 'get_parent_until', 'get_code',
'subscopes']:
raise AttributeError("Accessing %s of %s is not allowed."
% (self, name))
return getattr(self.func, name)
def __repr__(self):
return "<%s of %s>" % (type(self).__name__, self.func)
@@ -800,10 +804,8 @@ class Array(use_metaclass(cache.CachedMetaClass, pr.Base)):
if isinstance(index, Instance) \
and str(index.name) in ['int', 'str'] \
and len(index.var_args) == 1:
try:
with common.ignored(KeyError, IndexError):
return self.get_exact_index_types(index.var_args[0])
except (KeyError, IndexError):
pass
result = list(self._follow_values(self._array.values))
result += dynamic.check_array_additions(self)
jedi/helpers.py
+5 -6
View File
@@ -1,5 +1,8 @@
from __future__ import with_statement
import copy
from jedi import common
from jedi import parsing_representation as pr
@@ -21,13 +24,11 @@ def fast_parent_copy(obj):
before = ()
for cls in new_obj.__class__.__mro__:
try:
with common.ignored(AttributeError):
if before == cls.__slots__:
continue
before = cls.__slots__
items += [(n, getattr(new_obj, n)) for n in before]
except AttributeError:
pass
for key, value in items:
# replace parent (first try _parent and then parent)
@@ -35,10 +36,8 @@ def fast_parent_copy(obj):
if key == 'parent' and '_parent' in items:
# parent can be a property
continue
try:
with common.ignored(KeyError):
setattr(new_obj, key, new_elements[value])
except KeyError:
pass
elif key in ['parent_function', 'use_as_parent', '_sub_module']:
continue
elif isinstance(value, list):
jedi/imports.py
+19 -17
View File
@@ -5,23 +5,22 @@ any actual importing done. This module is about finding modules in the
filesystem. This can be quite tricky sometimes, because Python imports are not
always that simple.
Currently the import process uses ``imp`` to find modules. In the future, it's
a goal to use ``importlib`` for this purpose. There's a `pull request
<https://github.com/davidhalter/jedi/pull/109>`_ for that.
This module uses imp for python up to 3.2 and importlib for python 3.3 on; the
correct implementation is delegated to _compatibility.
This module also supports import autocompletion, which means to complete
statements like ``from datetim`` (curser at the end would return ``datetime``).
"""
from __future__ import with_statement
import os
import pkgutil
import imp
import sys
import itertools
from jedi._compatibility import find_module
from jedi import modules
from jedi import common
from jedi import debug
from jedi import parsing_representation as pr
from jedi import cache
@@ -123,11 +122,9 @@ class ImportPath(pr.Base):
if self.import_stmt.relative_count:
rel_path = self.get_relative_path() + '/__init__.py'
try:
with common.ignored(IOError):
m = modules.Module(rel_path)
names += m.parser.module.get_defined_names()
except IOError:
pass
else:
if on_import_stmt and isinstance(scope, pr.Module) \
and scope.path.endswith('__init__.py'):
@@ -238,20 +235,22 @@ class ImportPath(pr.Base):
global imports_processed
imports_processed += 1
importing = None
if path is not None:
return imp.find_module(string, [path])
importing = find_module(string, [path])
else:
debug.dbg('search_module', string, self.file_path)
# Override the sys.path. It works only good that way.
# Injecting the path directly into `find_module` did not work.
sys.path, temp = sys_path_mod, sys.path
try:
i = imp.find_module(string)
importing = find_module(string)
except ImportError:
sys.path = temp
raise
sys.path = temp
return i
return importing
if self.file_path:
sys_path_mod = list(self.sys_path_with_modifications())
@@ -259,6 +258,9 @@ class ImportPath(pr.Base):
else:
sys_path_mod = list(modules.get_sys_path())
def module_not_found():
raise ModuleNotFound('The module you searched has not been found')
current_namespace = (None, None, None)
# now execute those paths
rest = []
@@ -270,19 +272,19 @@ class ImportPath(pr.Base):
and len(self.import_path) == 1:
# follow `from . import some_variable`
rel_path = self.get_relative_path()
try:
with common.ignored(ImportError):
current_namespace = follow_str(rel_path, '__init__')
except ImportError:
pass
if current_namespace[1]:
rest = self.import_path[i:]
else:
raise ModuleNotFound(
'The module you searched has not been found')
module_not_found()
if current_namespace == (None, None, False):
module_not_found()
sys_path_mod.pop(0) # TODO why is this here?
path = current_namespace[1]
is_package_directory = current_namespace[2][2] == imp.PKG_DIRECTORY
is_package_directory = current_namespace[2]
f = None
if is_package_directory or current_namespace[0]:
jedi/keywords.py
+4 -3
View File
@@ -1,7 +1,10 @@
from __future__ import with_statement
import pydoc
import keyword
from jedi._compatibility import is_py3k
from jedi import common
import builtin
try:
@@ -63,12 +66,10 @@ def imitate_pydoc(string):
# with unicode strings)
string = str(string)
h = pydoc.help
try:
with common.ignored(KeyError):
# try to access symbols
string = h.symbols[string]
string, _, related = string.partition(' ')
except KeyError:
pass
get_target = lambda s: h.topics.get(s, h.keywords.get(s))
while isinstance(string, str):
jedi/modules.py
+4 -9
View File
@@ -27,6 +27,7 @@ from jedi import parsing_representation as pr
from jedi import fast_parser
from jedi import debug
from jedi import settings
from jedi import common
class CachedModule(object):
@@ -107,11 +108,9 @@ class ModuleWithCursor(Module):
def parser(self):
""" get the parser lazy """
if not self._parser:
try:
with common.ignored(KeyError):
parser = cache.parser_cache[self.path].parser
cache.invalidate_star_import_cache(parser.module)
except KeyError:
pass
# Call the parser already here, because it will be used anyways.
# Also, the position is here important (which will not be used by
# default), therefore fill the cache here.
@@ -348,10 +347,8 @@ def sys_path_with_modifications(module):
return [] # support for modules without a path is intentionally bad.
curdir = os.path.abspath(os.curdir)
try:
with common.ignored(OSError):
os.chdir(os.path.dirname(module.path))
except OSError:
pass
result = check_module(module)
result += detect_django_path(module.path)
@@ -372,12 +369,10 @@ def detect_django_path(module_path):
else:
module_path = new
try:
with common.ignored(IOError):
with open(module_path + os.path.sep + 'manage.py'):
debug.dbg('Found django path: %s' % module_path)
result.append(module_path)
except IOError:
pass
return result
jedi/parsing.py
+2 -3
View File
@@ -15,6 +15,7 @@ within the statement. This lowers memory usage and cpu time and reduces the
complexity of the ``Parser`` (there's another parser sitting inside
``Statement``, which produces ``Array`` and ``Call``).
"""
from __future__ import with_statement
import tokenize
import keyword
@@ -396,7 +397,7 @@ class Parser(object):
self._check_user_stmt(stmt)
# Attribute docstring (PEP 257) support
try:
with common.ignored(IndexError, AttributeError):
# If string literal is being parsed
first_tok = stmt.token_list[0]
if (not stmt.set_vars and
@@ -405,8 +406,6 @@ class Parser(object):
first_tok[0] == tokenize.STRING):
# ... then set it as a docstring
self.scope.statements[-1].add_docstr(first_tok[1])
except (IndexError, AttributeError):
pass
if tok in always_break + not_first_break:
self._gen.push_last_back()
jedi/parsing_representation.py
+5 -5
View File
@@ -34,6 +34,7 @@ statements in this scope. Check this out:
See also :attr:`Scope.subscopes` and :attr:`Scope.statements`.
"""
from __future__ import with_statement
import os
import re
@@ -560,8 +561,10 @@ class Flow(Scope):
@parent.setter
def parent(self, value):
self._parent = value
if self.next:
try:
self.next.parent = value
except AttributeError:
return
def get_code(self, first_indent=False, indention=' '):
stmts = []
@@ -1278,11 +1281,8 @@ class Array(Call):
inner = []
for i, stmt in enumerate(self.values):
s = ''
try:
with common.ignored(IndexError):
key = self.keys[i]
except IndexError:
pass
else:
s += key.get_code(new_line=False) + ': '
s += stmt.get_code(new_line=False)
inner.append(s)
jedi/refactoring.py
+2 -5
View File
@@ -12,11 +12,11 @@ following functions (sometimes bug-prone):
- extract variable
- inline variable
"""
from __future__ import with_statement
import difflib
from jedi import common
from jedi import modules
from jedi import helpers
from jedi import parsing_representation as pr
@@ -168,7 +168,7 @@ def inline(script):
dct = {}
definitions = script.goto()
try:
with common.ignored(AssertionError):
assert len(definitions) == 1
stmt = definitions[0].definition
related_names = script.related_names()
@@ -202,7 +202,4 @@ def inline(script):
else:
new_lines.pop(index)
except AssertionError:
pass
return Refactoring(dct)