From 70413768eff0fa2f1a98cc58891c4067f8c7f6d8 Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Tue, 7 Jan 2014 18:45:02 +0100 Subject: [PATCH 01/52] add a 'compiled' module, to finally solve #102 and #335 --- jedi/evaluate/compiled.py | 24 ++++++++++++++++++++++++ test/test_compiled.py | 10 ++++++++++ 2 files changed, 34 insertions(+) create mode 100644 jedi/evaluate/compiled.py create mode 100644 test/test_compiled.py diff --git a/jedi/evaluate/compiled.py b/jedi/evaluate/compiled.py new file mode 100644 index 00000000..f2f49837 --- /dev/null +++ b/jedi/evaluate/compiled.py @@ -0,0 +1,24 @@ +from jedi.cache import underscore_memoization + + +class PyObject(object): + def __init__(self, obj, parent=None): + self.obj = obj + self.parent = parent + + def get_defined_names(self): + for name in dir(self.obj): + yield PyName(self, name) + + +class PyName(object): + def __init__(self, obj, name): + self._obj = obj + self._name = name + + self.start_pos = 0, 0 # an illegal start_pos, to make sorting easy. + + @property + @underscore_memoization + def parent(self): + return PyObject(getattr(self._obj.obj, self._name), self._obj) diff --git a/test/test_compiled.py b/test/test_compiled.py new file mode 100644 index 00000000..45b64412 --- /dev/null +++ b/test/test_compiled.py @@ -0,0 +1,10 @@ +from jedi._compatibility import builtins +from jedi.evaluate import compiled +from jedi.evaluate import Evaluator + + +def test_simple(): + bltn = compiled.PyObject(builtins) + obj = compiled.PyObject('_str_', bltn) + upper = Evaluator().find_types(obj, 'upper') + assert len(upper) == 1 From f257e279c29077a520cd7c9b9aa84f3909dde812 Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Tue, 7 Jan 2014 18:48:54 +0100 Subject: [PATCH 02/52] pass the first test --- jedi/evaluate/compiled.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/jedi/evaluate/compiled.py b/jedi/evaluate/compiled.py index f2f49837..0ea587bf 100644 --- a/jedi/evaluate/compiled.py +++ b/jedi/evaluate/compiled.py @@ -1,3 +1,6 @@ +""" +Imitate the parser representation. +""" from jedi.cache import underscore_memoization @@ -10,6 +13,9 @@ class PyObject(object): for name in dir(self.obj): yield PyName(self, name) + def isinstance(self, *obj): + return isinstance(self, obj) + class PyName(object): def __init__(self, obj, name): @@ -22,3 +28,6 @@ class PyName(object): @underscore_memoization def parent(self): return PyObject(getattr(self._obj.obj, self._name), self._obj) + + def get_code(self): + return self._name From d71fe3061bad9ef4c29b59e712a86f83d8c7bf3d Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Wed, 8 Jan 2014 02:20:41 +0100 Subject: [PATCH 03/52] better environment for compiled executions --- jedi/evaluate/__init__.py | 5 ++++- jedi/evaluate/compiled.py | 33 +++++++++++++++++++++++++++++++-- test/test_compiled.py | 6 +++++- 3 files changed, 40 insertions(+), 4 deletions(-) diff --git a/jedi/evaluate/__init__.py b/jedi/evaluate/__init__.py index f08685b0..cde7e4ca 100644 --- a/jedi/evaluate/__init__.py +++ b/jedi/evaluate/__init__.py @@ -83,6 +83,7 @@ from jedi.evaluate import iterable from jedi.evaluate.cache import memoize_default from jedi.evaluate import stdlib from jedi.evaluate import finder +from jedi.evaluate import compiled class Evaluator(object): @@ -391,7 +392,9 @@ class Evaluator(object): except stdlib.NotInStdLib: pass - if obj.isinstance(er.Class): + if isinstance(obj, compiled.PyObject): + return obj.execute(params) + elif obj.isinstance(er.Class): # There maybe executions of executions. return [er.Instance(self, obj, params)] elif isinstance(obj, iterable.Generator): diff --git a/jedi/evaluate/compiled.py b/jedi/evaluate/compiled.py index 0ea587bf..5e716e63 100644 --- a/jedi/evaluate/compiled.py +++ b/jedi/evaluate/compiled.py @@ -1,13 +1,17 @@ """ Imitate the parser representation. """ +import inspect + +from jedi._compatibility import builtins from jedi.cache import underscore_memoization class PyObject(object): - def __init__(self, obj, parent=None): + def __init__(self, obj, parent=None, instantiated=False): self.obj = obj self.parent = parent + self.instantiated = instantiated def get_defined_names(self): for name in dir(self.obj): @@ -16,6 +20,21 @@ class PyObject(object): def isinstance(self, *obj): return isinstance(self, obj) + @property + def name(self): + # might not exist sometimes (raises AttributeError) + return self.obj.__name__ + + def execute(self, params): + if inspect.isclass(self.obj): + return [PyObject(self.obj, self.parent, True)] + elif inspect.isbuiltin(self.obj) or inspect.ismethod(self.obj) \ + or inspect.ismethoddescriptor(self.obj): + return [] + else: + return [] + return [] + class PyName(object): def __init__(self, obj, name): @@ -27,7 +46,17 @@ class PyName(object): @property @underscore_memoization def parent(self): - return PyObject(getattr(self._obj.obj, self._name), self._obj) + try: + # this has a builtin_function_or_method + o = getattr(self._obj.obj, self._name) + except AttributeError: + # happens e.g. in properties of + # PyQt4.QtGui.QStyleOptionComboBox.currentText + # -> just set it to None + return PyObject(obj, py_builtin) + return PyObject(o, self._obj) def get_code(self): return self._name + +py_builtin = PyObject(builtins) diff --git a/test/test_compiled.py b/test/test_compiled.py index 45b64412..cb08d0eb 100644 --- a/test/test_compiled.py +++ b/test/test_compiled.py @@ -4,7 +4,11 @@ from jedi.evaluate import Evaluator def test_simple(): + e = Evaluator() bltn = compiled.PyObject(builtins) obj = compiled.PyObject('_str_', bltn) - upper = Evaluator().find_types(obj, 'upper') + upper = e.find_types(obj, 'upper') assert len(upper) == 1 + objs = e.execute(upper[0]) + assert len(objs) == 1 + assert isinstance(objs[0].obj, str) From 7af9e6dfd7cd51be6333667b6e44a571a98dec93 Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Wed, 8 Jan 2014 22:01:26 +0100 Subject: [PATCH 04/52] pushed the _parse_function_doc to compiled --- jedi/evaluate/compiled.py | 114 +++++++++++++++++++++++++++++++++++--- 1 file changed, 107 insertions(+), 7 deletions(-) diff --git a/jedi/evaluate/compiled.py b/jedi/evaluate/compiled.py index 5e716e63..b5250110 100644 --- a/jedi/evaluate/compiled.py +++ b/jedi/evaluate/compiled.py @@ -2,20 +2,40 @@ Imitate the parser representation. """ import inspect +import re -from jedi._compatibility import builtins +from jedi._compatibility import builtins as _builtins, is_py3k +from jedi import debug from jedi.cache import underscore_memoization +# TODO +# unbound methods such as pyqtSignals have no __name__ +# if not hasattr(func, "__name__"): + class PyObject(object): def __init__(self, obj, parent=None, instantiated=False): self.obj = obj self.parent = parent self.instantiated = instantiated + self.doc = inspect.getdoc(obj) + + @underscore_memoization + def _parse_function_doc(self): + if self.doc is None: + return '', '' + + return _parse_function_doc(self.doc) def get_defined_names(self): - for name in dir(self.obj): - yield PyName(self, name) + # We don't want to execute properties, therefore we have to try to get + # the class + cls = self + if not (inspect.isclass(cls) or inspect.ismodule(cls)): + cls = PyObject(self.obj.__class__, self.parent) + + for name in dir(cls): + yield PyName(cls, name) def isinstance(self, *obj): return isinstance(self, obj) @@ -30,6 +50,8 @@ class PyObject(object): return [PyObject(self.obj, self.parent, True)] elif inspect.isbuiltin(self.obj) or inspect.ismethod(self.obj) \ or inspect.ismethoddescriptor(self.obj): + + self.doc return [] else: return [] @@ -48,15 +70,93 @@ class PyName(object): def parent(self): try: # this has a builtin_function_or_method - o = getattr(self._obj.obj, self._name) + return PyObject(getattr(self._obj.obj, self._name), self._obj) except AttributeError: # happens e.g. in properties of # PyQt4.QtGui.QStyleOptionComboBox.currentText # -> just set it to None - return PyObject(obj, py_builtin) - return PyObject(o, self._obj) + return PyObject(None, builtin) def get_code(self): return self._name -py_builtin = PyObject(builtins) + +docstr_defaults = { + 'floating point number': 'float', + 'character': 'str', + 'integer': 'int', + 'dictionary': 'dict', +} + +if is_py3k: + #docstr_defaults['file object'] = 'import io; return io.TextIOWrapper()' + pass # TODO reenable +else: + docstr_defaults['file object'] = file + + +def _parse_function_doc(doc): + """ + Takes a function and returns the params and return value as a tuple. + This is nothing more than a docstring parser. + + TODO docstrings like utime(path, (atime, mtime)) and a(b [, b]) -> None + TODO docstrings like 'tuple of integers' + """ + + # parse round parentheses: def func(a, (b,c)) + try: + count = 0 + start = doc.index('(') + for i, s in enumerate(doc[start:]): + if s == '(': + count += 1 + elif s == ')': + count -= 1 + if count == 0: + end = start + i + break + param_str = doc[start + 1:end] + except (ValueError, UnboundLocalError): + # ValueError for doc.index + # UnboundLocalError for undefined end in last line + debug.dbg('no brackets found - no param') + end = 0 + param_str = '' + else: + # remove square brackets, that show an optional param ( = None) + def change_options(m): + args = m.group(1).split(',') + for i, a in enumerate(args): + if a and '=' not in a: + args[i] += '=None' + return ','.join(args) + + while True: + param_str, changes = re.subn(r' ?\[([^\[\]]+)\]', + change_options, param_str) + if changes == 0: + break + param_str = param_str.replace('-', '_') # see: isinstance.__doc__ + + # parse return value + r = re.search('-[>-]* ', doc[end:end + 7]) + if r is None: + ret = '' + else: + index = end + r.end() + # get result type, which can contain newlines + pattern = re.compile(r'(,\n|[^\n-])+') + ret_str = pattern.match(doc, index).group(0).strip() + # New object -> object() + ret_str = re.sub(r'[nN]ew (.*)', r'\1()', ret_str) + + ret = docstr_defaults.get(ret_str, ret_str) + if ret == ret_str and ret not in ['None', 'object', 'tuple', 'set']: + debug.dbg('not working', ret_str) + + ret = ('return ' if 'return' not in ret else '') + ret + return param_str, ret + + +builtin = PyObject(_builtins) From 3017e72b86090ef6e82f1cc2f57819310967adc8 Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Thu, 9 Jan 2014 00:53:50 +0100 Subject: [PATCH 05/52] first executions with compiled seem to be working --- jedi/evaluate/compiled.py | 26 +++++++++++++------------- test/test_compiled.py | 5 +++-- 2 files changed, 16 insertions(+), 15 deletions(-) diff --git a/jedi/evaluate/compiled.py b/jedi/evaluate/compiled.py index b5250110..5da33cd2 100644 --- a/jedi/evaluate/compiled.py +++ b/jedi/evaluate/compiled.py @@ -20,6 +20,9 @@ class PyObject(object): self.instantiated = instantiated self.doc = inspect.getdoc(obj) + def __repr__(self): + return '<%s: %s>' % (type(self).__name__, self.obj) + @underscore_memoization def _parse_function_doc(self): if self.doc is None: @@ -31,10 +34,10 @@ class PyObject(object): # We don't want to execute properties, therefore we have to try to get # the class cls = self - if not (inspect.isclass(cls) or inspect.ismodule(cls)): + if not (inspect.isclass(self.obj) or inspect.ismodule(self.obj)): cls = PyObject(self.obj.__class__, self.parent) - for name in dir(cls): + for name in dir(cls.obj): yield PyName(cls, name) def isinstance(self, *obj): @@ -47,15 +50,14 @@ class PyObject(object): def execute(self, params): if inspect.isclass(self.obj): - return [PyObject(self.obj, self.parent, True)] + yield PyObject(self.obj, self.parent, True) elif inspect.isbuiltin(self.obj) or inspect.ismethod(self.obj) \ or inspect.ismethoddescriptor(self.obj): - - self.doc - return [] - else: - return [] - return [] + for name in self._parse_function_doc()[1].split(): + try: + yield PyObject(getattr(_builtins, name), builtin, True) + except AttributeError: + pass class PyName(object): @@ -86,13 +88,14 @@ docstr_defaults = { 'character': 'str', 'integer': 'int', 'dictionary': 'dict', + 'string': 'str', } if is_py3k: #docstr_defaults['file object'] = 'import io; return io.TextIOWrapper()' pass # TODO reenable else: - docstr_defaults['file object'] = file + docstr_defaults['file object'] = 'file' def _parse_function_doc(doc): @@ -152,10 +155,7 @@ def _parse_function_doc(doc): ret_str = re.sub(r'[nN]ew (.*)', r'\1()', ret_str) ret = docstr_defaults.get(ret_str, ret_str) - if ret == ret_str and ret not in ['None', 'object', 'tuple', 'set']: - debug.dbg('not working', ret_str) - ret = ('return ' if 'return' not in ret else '') + ret return param_str, ret diff --git a/test/test_compiled.py b/test/test_compiled.py index cb08d0eb..930c27e4 100644 --- a/test/test_compiled.py +++ b/test/test_compiled.py @@ -9,6 +9,7 @@ def test_simple(): obj = compiled.PyObject('_str_', bltn) upper = e.find_types(obj, 'upper') assert len(upper) == 1 - objs = e.execute(upper[0]) + objs = list(e.execute(upper[0])) assert len(objs) == 1 - assert isinstance(objs[0].obj, str) + assert objs[0].obj is str + assert objs[0].instantiated is True From 0cb23dcfa2e54481c3d96c9fb281641a89a86cd4 Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Thu, 9 Jan 2014 01:26:51 +0100 Subject: [PATCH 06/52] tried to start introducing the compiled module to the library --- jedi/api/classes.py | 18 ++++++++---------- jedi/evaluate/__init__.py | 3 +-- jedi/evaluate/compiled.py | 22 ++++++++++++++++------ 3 files changed, 25 insertions(+), 18 deletions(-) diff --git a/jedi/api/classes.py b/jedi/api/classes.py index 341882f2..6328808a 100644 --- a/jedi/api/classes.py +++ b/jedi/api/classes.py @@ -14,6 +14,7 @@ from jedi import cache from jedi.evaluate import representation as er from jedi.evaluate import iterable from jedi.evaluate import imports +from jedi.evaluate import compiled from jedi import keywords @@ -70,8 +71,11 @@ class BaseDefinition(object): # generate a path to the definition self._module = definition.get_parent_until() - self.module_path = self._module.path - """Shows the file path of a module. e.g. ``/usr/lib/python2.7/os.py``""" + if self.in_builtin_module(): + self.module_path = None + else: + self.module_path = self._module.path + """Shows the file path of a module. e.g. ``/usr/lib/python2.7/os.py``""" @property def start_pos(self): @@ -177,8 +181,7 @@ class BaseDefinition(object): def in_builtin_module(self): """Whether this is a builtin module.""" - return not (self.module_path is None or - self.module_path.endswith('.py')) + return isinstance(self._module, compiled.PyObject) @property def line_nr(self): @@ -518,12 +521,7 @@ class Definition(BaseDefinition): .. todo:: Add full path. This function is should return a `module.class.function` path. """ - if self.module_path.endswith('.py') \ - and not isinstance(self._definition, pr.Module): - position = '@%s' % (self.line) - else: - # is a builtin or module - position = '' + position = '' if self.in_builtin_module else '@%s' % (self.line) return "%s:%s%s" % (self.module_name, self.description, position) def defined_names(self): diff --git a/jedi/evaluate/__init__.py b/jedi/evaluate/__init__.py index cde7e4ca..3b2fe815 100644 --- a/jedi/evaluate/__init__.py +++ b/jedi/evaluate/__init__.py @@ -174,8 +174,7 @@ class Evaluator(object): # Add builtins to the global scope. if include_builtin: - builtin_scope = builtin.Builtin.scope - yield builtin_scope, builtin_scope.get_defined_names() + yield compiled.builtin, compiled.builtin.get_defined_names() def find_types(self, scope, name_str, position=None, search_global=False, is_goto=False, resolve_decorator=True): diff --git a/jedi/evaluate/compiled.py b/jedi/evaluate/compiled.py index 5da33cd2..6c0b876a 100644 --- a/jedi/evaluate/compiled.py +++ b/jedi/evaluate/compiled.py @@ -6,6 +6,7 @@ import re from jedi._compatibility import builtins as _builtins, is_py3k from jedi import debug +from jedi.parser.representation import Base from jedi.cache import underscore_memoization @@ -13,13 +14,17 @@ from jedi.cache import underscore_memoization # unbound methods such as pyqtSignals have no __name__ # if not hasattr(func, "__name__"): -class PyObject(object): +class PyObject(Base): def __init__(self, obj, parent=None, instantiated=False): self.obj = obj self.parent = parent self.instantiated = instantiated self.doc = inspect.getdoc(obj) + # comply with the parser + self.get_parent_until = lambda: parent + self.start_pos = 0, 0 + def __repr__(self): return '<%s: %s>' % (type(self).__name__, self.obj) @@ -30,19 +35,25 @@ class PyObject(object): return _parse_function_doc(self.doc) + def type(self): + if inspect.isclass(self.obj): + return 'class' + elif inspect.ismodule(self.obj): + return 'module' + elif inspect.isbuiltin(self.obj) or inspect.ismethod(self.obj) \ + or inspect.ismethoddescriptor(self.obj): + return 'def' + def get_defined_names(self): # We don't want to execute properties, therefore we have to try to get # the class cls = self - if not (inspect.isclass(self.obj) or inspect.ismodule(self.obj)): + if not inspect.isclass(self.obj): cls = PyObject(self.obj.__class__, self.parent) for name in dir(cls.obj): yield PyName(cls, name) - def isinstance(self, *obj): - return isinstance(self, obj) - @property def name(self): # might not exist sometimes (raises AttributeError) @@ -64,7 +75,6 @@ class PyName(object): def __init__(self, obj, name): self._obj = obj self._name = name - self.start_pos = 0, 0 # an illegal start_pos, to make sorting easy. @property From e7e802408b66dfb78d89fde3a5ee806d4dc70f14 Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Thu, 9 Jan 2014 01:52:10 +0100 Subject: [PATCH 07/52] fix some api stuff for PyObject --- jedi/api/classes.py | 4 +++- jedi/evaluate/compiled.py | 12 ++++++++---- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/jedi/api/classes.py b/jedi/api/classes.py index 6328808a..bca6975a 100644 --- a/jedi/api/classes.py +++ b/jedi/api/classes.py @@ -493,7 +493,9 @@ class Definition(BaseDefinition): if isinstance(d, pr.Name): d = d.parent - if isinstance(d, iterable.Array): + if isinstance(d, compiled.PyObject): + d = d.type() + ' ' + d.name + elif isinstance(d, iterable.Array): d = 'class ' + d.type elif isinstance(d, (pr.Class, er.Class, er.Instance)): d = 'class ' + unicode(d.name) diff --git a/jedi/evaluate/compiled.py b/jedi/evaluate/compiled.py index 6c0b876a..2d10b703 100644 --- a/jedi/evaluate/compiled.py +++ b/jedi/evaluate/compiled.py @@ -43,12 +43,13 @@ class PyObject(Base): elif inspect.isbuiltin(self.obj) or inspect.ismethod(self.obj) \ or inspect.ismethoddescriptor(self.obj): return 'def' + raise NotImplementedError() def get_defined_names(self): # We don't want to execute properties, therefore we have to try to get # the class cls = self - if not inspect.isclass(self.obj): + if self.type() not in ('class', 'module'): cls = PyObject(self.obj.__class__, self.parent) for name in dir(cls.obj): @@ -60,10 +61,10 @@ class PyObject(Base): return self.obj.__name__ def execute(self, params): - if inspect.isclass(self.obj): + t = self.type() + if t == 'class': yield PyObject(self.obj, self.parent, True) - elif inspect.isbuiltin(self.obj) or inspect.ismethod(self.obj) \ - or inspect.ismethoddescriptor(self.obj): + elif t == 'def': for name in self._parse_function_doc()[1].split(): try: yield PyObject(getattr(_builtins, name), builtin, True) @@ -77,6 +78,9 @@ class PyName(object): self._name = name self.start_pos = 0, 0 # an illegal start_pos, to make sorting easy. + def __repr__(self): + return '<%s: %s.%s>' % (type(self).__name__, self._obj.obj, self._name) + @property @underscore_memoization def parent(self): From df6317f8b04b353409e1ae494c4215290136692f Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Thu, 9 Jan 2014 01:56:27 +0100 Subject: [PATCH 08/52] already a lot of tests pass - time to kick it up a notch --- jedi/evaluate/compiled.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/jedi/evaluate/compiled.py b/jedi/evaluate/compiled.py index 2d10b703..d901aa9f 100644 --- a/jedi/evaluate/compiled.py +++ b/jedi/evaluate/compiled.py @@ -93,6 +93,10 @@ class PyName(object): # -> just set it to None return PyObject(None, builtin) + @property + def names(self): + return [self._name] # compatibility with parser.representation.Name + def get_code(self): return self._name From d2358c60b71a1cb6516f600d797407f8234e72b5 Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Thu, 9 Jan 2014 02:02:33 +0100 Subject: [PATCH 09/52] more builtin replacements with compiled --- jedi/evaluate/__init__.py | 3 +-- jedi/evaluate/finder.py | 7 ++++--- jedi/evaluate/representation.py | 10 +++++----- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/jedi/evaluate/__init__.py b/jedi/evaluate/__init__.py index 3b2fe815..bd0690b4 100644 --- a/jedi/evaluate/__init__.py +++ b/jedi/evaluate/__init__.py @@ -76,7 +76,6 @@ from jedi import common from jedi.parser import representation as pr from jedi import debug from jedi.evaluate import representation as er -from jedi.evaluate import builtin from jedi.evaluate import imports from jedi.evaluate import recursion from jedi.evaluate import iterable @@ -312,7 +311,7 @@ class Evaluator(object): search_global=True) else: # for pr.Literal - scopes = self.find_types(builtin.Builtin.scope, current.type_as_string()) + scopes = self.find_types(compiled.builtin, current.type_as_string()) # Make instances of those number/string objects. scopes = itertools.chain.from_iterable( self.execute(s, (current.value,)) for s in scopes diff --git a/jedi/evaluate/finder.py b/jedi/evaluate/finder.py index a03ec72c..96e15d9f 100644 --- a/jedi/evaluate/finder.py +++ b/jedi/evaluate/finder.py @@ -7,7 +7,7 @@ from jedi import common from jedi import settings from jedi.evaluate import representation as er from jedi.evaluate import dynamic -from jedi.evaluate import builtin +from jedi.evaluate import compiled from jedi.evaluate import docstrings from jedi.evaluate import iterable @@ -82,7 +82,7 @@ class NameFinder(object): def _check_getattr(self, inst): """Checks for both __getattr__ and __getattribute__ methods""" result = [] - module = builtin.Builtin.scope + module = compiled.builtin # str is important to lose the NamePart! name = pr.String(module, "'%s'" % self.name_str, (0, 0), (0, 0), inst) with common.ignored(KeyError): @@ -242,7 +242,8 @@ class NameFinder(object): c = r.expression_list()[0] if c in ('*', '**'): t = 'tuple' if c == '*' else 'dict' - res_new = evaluator.execute(evaluator.find_types(builtin.Builtin.scope, t)[0]) + typ = evaluator.find_types(compiled.builtin, t)[0] + res_new = evaluator.execute(typ) if not r.assignment_details: # this means that there are no default params, # so just ignore it. diff --git a/jedi/evaluate/representation.py b/jedi/evaluate/representation.py index f9ea1f37..1736e721 100644 --- a/jedi/evaluate/representation.py +++ b/jedi/evaluate/representation.py @@ -16,7 +16,7 @@ from jedi.parser import representation as pr from jedi import debug from jedi import common from jedi.evaluate.cache import memoize_default, CachedMetaClass -from jedi.evaluate import builtin +from jedi.evaluate import compiled from jedi.evaluate import recursion from jedi.evaluate import iterable from jedi.evaluate import docstrings @@ -49,7 +49,7 @@ class Instance(use_metaclass(CachedMetaClass, Executable)): def __init__(self, evaluator, base, var_args=()): super(Instance, self).__init__(evaluator, base, var_args) if str(base.name) in ['list', 'set'] \ - and builtin.Builtin.scope == base.get_parent_until(): + and compiled.builtin == base.get_parent_until(): # compare the module path with the builtin name. self.var_args = iterable.check_array_instances(evaluator, self) else: @@ -256,9 +256,9 @@ class Class(use_metaclass(CachedMetaClass, pr.IsScope)): debug.warning('Received non class, as a super class') continue # Just ignore other stuff (user input error). supers.append(cls) - if not supers and self.base.parent != builtin.Builtin.scope: + if not supers and self.base.parent != compiled.builtin: # add `object` to classes - supers += self._evaluator.find_types(builtin.Builtin.scope, 'object') + supers += self._evaluator.find_types(compiled.builtin, 'object') return supers @memoize_default(default=()) @@ -286,7 +286,7 @@ class Class(use_metaclass(CachedMetaClass, pr.IsScope)): @memoize_default(default=()) def get_defined_names(self): result = self.instance_names() - type_cls = self._evaluator.find_types(builtin.Builtin.scope, 'type')[0] + type_cls = self._evaluator.find_types(compiled.builtin, 'type')[0] return result + type_cls.base.get_defined_names() def get_subscope_by_name(self, name): From 11e2446438ff983066901a2dc1cc24a1cde6ff0f Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Thu, 9 Jan 2014 11:05:04 +0100 Subject: [PATCH 10/52] replaced builtin with compiled in all modules except imports --- jedi/api/__init__.py | 9 ++++----- jedi/evaluate/compiled.py | 4 ++++ jedi/evaluate/iterable.py | 14 +++++++------- jedi/evaluate/param.py | 5 ++--- jedi/evaluate/recursion.py | 6 +++--- jedi/evaluate/representation.py | 22 ++++++++++++---------- jedi/evaluate/stdlib.py | 4 ++-- jedi/keywords.py | 4 ++-- 8 files changed, 36 insertions(+), 32 deletions(-) diff --git a/jedi/api/__init__.py b/jedi/api/__init__.py index b78a9d11..ce50e546 100644 --- a/jedi/api/__init__.py +++ b/jedi/api/__init__.py @@ -27,7 +27,7 @@ from jedi import keywords from jedi.api import classes from jedi.evaluate import Evaluator, filter_private_variable from jedi.evaluate import representation as er -from jedi.evaluate import builtin +from jedi.evaluate import compiled from jedi.evaluate import imports from jedi.evaluate import helpers @@ -137,8 +137,8 @@ class Script(object): path, dot, like = self._get_completion_parts() user_stmt = self._user_stmt(True) - bs = builtin.Builtin.scope - completions = get_completions(user_stmt, bs) + b = compiled.builtin + completions = get_completions(user_stmt, b) if not dot: # add named params @@ -149,8 +149,7 @@ class Script(object): if not path and not isinstance(user_stmt, pr.Import): # add keywords - completions += ((k, bs) for k in keywords.keyword_names( - all=True)) + completions += ((k, b) for k in keywords.keyword_names(all=True)) needs_dot = not dot and path diff --git a/jedi/evaluate/compiled.py b/jedi/evaluate/compiled.py index d901aa9f..8af613ee 100644 --- a/jedi/evaluate/compiled.py +++ b/jedi/evaluate/compiled.py @@ -71,6 +71,10 @@ class PyObject(Base): except AttributeError: pass + def get_self_attributes(self): + # Instance compatibility + return [] + class PyName(object): def __init__(self, obj, name): diff --git a/jedi/evaluate/iterable.py b/jedi/evaluate/iterable.py index ba237fa1..4fab6c1d 100644 --- a/jedi/evaluate/iterable.py +++ b/jedi/evaluate/iterable.py @@ -5,7 +5,7 @@ from jedi import debug from jedi import settings from jedi._compatibility import use_metaclass, is_py3k from jedi.parser import representation as pr -from jedi.evaluate import builtin +from jedi.evaluate import compiled from jedi.evaluate import helpers from jedi.evaluate.cache import CachedMetaClass, memoize_default @@ -27,12 +27,12 @@ class Generator(use_metaclass(CachedMetaClass, pr.Base)): none_pos = (0, 0) executes_generator = ('__next__', 'send') for n in ('close', 'throw') + executes_generator: - name = pr.Name(builtin.Builtin.scope, [(n, none_pos)], + name = pr.Name(compiled.builtin, [(n, none_pos)], none_pos, none_pos) if n in executes_generator: name.parent = self else: - name.parent = builtin.Builtin.scope + name.parent = compiled.builtin names.append(name) debug.dbg('generator names', names) return names @@ -130,17 +130,17 @@ class Array(use_metaclass(CachedMetaClass, pr.Base)): It returns e.g. for a list: append, pop, ... """ # `array.type` is a string with the type, e.g. 'list'. - scope = self._evaluator.find_types(builtin.Builtin.scope, self._array.type)[0] + scope = self._evaluator.find_types(compiled.builtin, self._array.type)[0] scope = self._evaluator.execute(scope)[0] # builtins only have one class names = scope.get_defined_names() return [ArrayMethod(n) for n in names] @property def parent(self): - return builtin.Builtin.scope + return compiled.builtin def get_parent_until(self): - return builtin.Builtin.scope + return compiled.builtin def __getattr__(self, name): if name not in ['type', 'start_pos', 'get_only_subelement', 'parent', @@ -177,7 +177,7 @@ class ArrayMethod(object): return getattr(self.name, name) def get_parent_until(self): - return builtin.Builtin.scope + return compiled.builtin def __repr__(self): return "<%s of %s>" % (type(self).__name__, self.name) diff --git a/jedi/evaluate/param.py b/jedi/evaluate/param.py index 449aa513..eb4f59eb 100644 --- a/jedi/evaluate/param.py +++ b/jedi/evaluate/param.py @@ -2,7 +2,7 @@ import copy from jedi.parser import representation as pr from jedi.evaluate import iterable -from jedi.evaluate import builtin +from jedi.evaluate import compiled from jedi.evaluate import common @@ -140,8 +140,7 @@ def _var_args_iterator(evaluator, var_args): continue old = stmt # generate a statement if it's not already one. - module = builtin.Builtin.scope - stmt = pr.Statement(module, [], (0, 0), None) + stmt = pr.Statement(compiled.builtin, [], (0, 0), None) stmt._expression_list = [old] # *args diff --git a/jedi/evaluate/recursion.py b/jedi/evaluate/recursion.py index 51bb598e..620ed93e 100644 --- a/jedi/evaluate/recursion.py +++ b/jedi/evaluate/recursion.py @@ -10,7 +10,7 @@ calls. from jedi.parser import representation as pr from jedi import debug from jedi import settings -from jedi.evaluate import builtin +from jedi.evaluate import compiled from jedi.evaluate import iterable @@ -82,7 +82,7 @@ class _RecursionNode(object): # The same's true for the builtins, because the builtins are really # simple. self.is_ignored = isinstance(stmt, pr.Param) \ - or (self.script == builtin.Builtin.scope) + or (self.script == compiled.builtin) def __eq__(self, other): if not other: @@ -148,7 +148,7 @@ class ExecutionRecursionDetector(object): if isinstance(execution.base, (iterable.Array, iterable.Generator)): return False module = execution.get_parent_until() - if evaluate_generator or module == builtin.Builtin.scope: + if evaluate_generator or module == compiled.builtin: return False if in_par_execution_funcs: diff --git a/jedi/evaluate/representation.py b/jedi/evaluate/representation.py index 1736e721..5a9d39e3 100644 --- a/jedi/evaluate/representation.py +++ b/jedi/evaluate/representation.py @@ -141,8 +141,7 @@ class Instance(use_metaclass(CachedMetaClass, Executable)): """ names = self.get_self_attributes() - class_names = self.base.instance_names() - for var in class_names: + for var in self.base.instance_names(): names.append(InstanceElement(self._evaluator, self, var, True)) return names @@ -154,8 +153,7 @@ class Instance(use_metaclass(CachedMetaClass, Executable)): yield self, self.get_self_attributes() names = [] - class_names = self.base.instance_names() - for var in class_names: + for var in self.base.instance_names(): names.append(InstanceElement(self._evaluator, self, var, True)) yield self, names @@ -263,6 +261,7 @@ class Class(use_metaclass(CachedMetaClass, pr.IsScope)): @memoize_default(default=()) def instance_names(self): + # TODO REMOVE instance_names def in_iterable(name, iterable): """ checks if the name is in the variable 'iterable'. """ for i in iterable: @@ -277,9 +276,12 @@ class Class(use_metaclass(CachedMetaClass, pr.IsScope)): # TODO mro! for cls in self.get_super_classes(): # Get the inherited names. - for i in cls.instance_names(): - if not in_iterable(i, result): - super_result.append(i) + if isinstance(cls, compiled.PyObject): + super_result += cls.get_defined_names() + else: + for i in cls.instance_names(): + if not in_iterable(i, result): + super_result.append(i) result += super_result return result @@ -287,7 +289,7 @@ class Class(use_metaclass(CachedMetaClass, pr.IsScope)): def get_defined_names(self): result = self.instance_names() type_cls = self._evaluator.find_types(compiled.builtin, 'type')[0] - return result + type_cls.base.get_defined_names() + return result + list(type_cls.get_defined_names()) def get_subscope_by_name(self, name): for sub in reversed(self.subscopes): @@ -374,10 +376,10 @@ class Function(use_metaclass(CachedMetaClass, pr.IsScope)): return decorated_func def get_magic_method_names(self): - return builtin.Builtin.magic_function_scope(self._evaluator).get_defined_names() + return compiled.magic_function_scope(self._evaluator).get_defined_names() def get_magic_method_scope(self): - return builtin.Builtin.magic_function_scope(self._evaluator) + return compiled.magic_function_scope(self._evaluator) def __getattr__(self, name): return getattr(self.base_func, name) diff --git a/jedi/evaluate/stdlib.py b/jedi/evaluate/stdlib.py index 25857d9f..4d821579 100644 --- a/jedi/evaluate/stdlib.py +++ b/jedi/evaluate/stdlib.py @@ -3,7 +3,7 @@ Implementations of standard library functions, because it's not possible to understand them with Jedi. """ -from jedi.evaluate import builtin +from jedi.evaluate import compiled from jedi.evaluate import representation as er from jedi.evaluate import iterable from jedi.parser import representation as pr @@ -17,7 +17,7 @@ class NotInStdLib(LookupError): def execute(evaluator, obj, params): if not isinstance(obj, (iterable.Generator, iterable.Array)): obj_name = str(obj.name) - if obj.parent == builtin.Builtin.scope: + if obj.parent == compiled.builtin: # for now we just support builtin functions. try: return _implemented['builtins'][obj_name](evaluator, obj, params) diff --git a/jedi/keywords.py b/jedi/keywords.py index ab0fd4b3..df26202a 100644 --- a/jedi/keywords.py +++ b/jedi/keywords.py @@ -4,7 +4,7 @@ import keyword from jedi._compatibility import is_py3k from jedi.parser import representation as pr from jedi import common -from jedi.evaluate import builtin +from jedi.evaluate import compiled try: from pydoc_data import topics as pydoc_topics @@ -43,7 +43,7 @@ class Keyword(object): def __init__(self, name, pos): self.name = name self.start_pos = pos - self.parent = builtin.Builtin.scope + self.parent = compiled.builtin def get_parent_until(self): return self.parent From 7965cae3737328ae6e56c6de5982deb855cd48cd Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Thu, 9 Jan 2014 14:38:15 +0100 Subject: [PATCH 11/52] also changed the imports stuff to partially support compiled --- jedi/evaluate/compiled.py | 31 +++++++++++++++++++++++++++---- jedi/evaluate/imports.py | 10 +++++----- 2 files changed, 32 insertions(+), 9 deletions(-) diff --git a/jedi/evaluate/compiled.py b/jedi/evaluate/compiled.py index 8af613ee..661ec79a 100644 --- a/jedi/evaluate/compiled.py +++ b/jedi/evaluate/compiled.py @@ -3,11 +3,13 @@ Imitate the parser representation. """ import inspect import re +import sys -from jedi._compatibility import builtins as _builtins, is_py3k +from jedi._compatibility import builtins as _builtins, is_py3k, exec_function from jedi import debug from jedi.parser.representation import Base from jedi.cache import underscore_memoization +from jedi.evaluate.sys_path import get_sys_path # TODO @@ -22,7 +24,7 @@ class PyObject(Base): self.doc = inspect.getdoc(obj) # comply with the parser - self.get_parent_until = lambda: parent + self.get_parent_until = lambda *args, **kwargs: parent self.start_pos = 0, 0 def __repr__(self): @@ -72,8 +74,10 @@ class PyObject(Base): pass def get_self_attributes(self): - # Instance compatibility - return [] + return [] # Instance compatibility + + def get_imports(self): + return [] # Builtins don't have imports class PyName(object): @@ -105,6 +109,25 @@ class PyName(object): return self._name +def load_module(path, name): + sys_path = get_sys_path() + if path: + sys_path.insert(0, path) + + temp, sys.path = sys.path, sys_path + content = {} + try: + exec_function('import %s as module' % name, content) + module = content['module'] + except AttributeError: + # use sys.modules, because you cannot access some modules + # directly. -> github issue #59 + module = sys.modules[name] + sys.path = temp + + return PyObject(module) + + docstr_defaults = { 'floating point number': 'float', 'character': 'str', diff --git a/jedi/evaluate/imports.py b/jedi/evaluate/imports.py index 7176609d..d1020615 100644 --- a/jedi/evaluate/imports.py +++ b/jedi/evaluate/imports.py @@ -25,6 +25,7 @@ from jedi.parser import representation as pr from jedi.evaluate import sys_path from jedi import settings from jedi.common import source_to_unicode +from jedi.evaluate import compiled class ModuleNotFound(Exception): @@ -381,8 +382,9 @@ def strip_imports(evaluator, scopes): @cache.cache_star_import def remove_star_imports(evaluator, scope, ignored_modules=()): """ - Check a module for star imports: - >>> from module import * + Check a module for star imports:: + + from module import * and follow these modules. """ @@ -404,9 +406,7 @@ def load_module(path=None, source=None, name=None): with open(path) as f: source = f.read() else: - # TODO refactoring remove - from jedi.evaluate import builtin - return builtin.BuiltinModule(path, name).parser.module + return compiled.load_module(path, name) p = path or name p = fast.FastParser(common.source_to_unicode(source), p) cache.save_parser(path, name, p) From db149a84a85fd1173665665dcfa60b8cf45e9bd3 Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Thu, 9 Jan 2014 16:50:31 +0100 Subject: [PATCH 12/52] fixed a lot of import related problems --- jedi/evaluate/compiled.py | 31 +++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/jedi/evaluate/compiled.py b/jedi/evaluate/compiled.py index 661ec79a..c70d0b7a 100644 --- a/jedi/evaluate/compiled.py +++ b/jedi/evaluate/compiled.py @@ -4,6 +4,7 @@ Imitate the parser representation. import inspect import re import sys +import os from jedi._compatibility import builtins as _builtins, is_py3k, exec_function from jedi import debug @@ -24,7 +25,7 @@ class PyObject(Base): self.doc = inspect.getdoc(obj) # comply with the parser - self.get_parent_until = lambda *args, **kwargs: parent + self.get_parent_until = lambda *args, **kwargs: parent or self self.start_pos = 0, 0 def __repr__(self): @@ -38,22 +39,24 @@ class PyObject(Base): return _parse_function_doc(self.doc) def type(self): - if inspect.isclass(self.obj): + cls = self._cls().obj + if inspect.isclass(cls): return 'class' - elif inspect.ismodule(self.obj): + elif inspect.ismodule(cls): return 'module' - elif inspect.isbuiltin(self.obj) or inspect.ismethod(self.obj) \ - or inspect.ismethoddescriptor(self.obj): + elif inspect.isbuiltin(cls) or inspect.ismethod(cls) \ + or inspect.ismethoddescriptor(cls): return 'def' - raise NotImplementedError() + + @underscore_memoization + def _cls(self): + # Ensures that a PyObject is returned that is not an instance (like list) + if not (inspect.isclass(self.obj) or inspect.ismodule(self.obj)): + return PyObject(self.obj.__class__, self.parent, True) + return self def get_defined_names(self): - # We don't want to execute properties, therefore we have to try to get - # the class - cls = self - if self.type() not in ('class', 'module'): - cls = PyObject(self.obj.__class__, self.parent) - + cls = self._cls() for name in dir(cls.obj): yield PyName(cls, name) @@ -110,6 +113,10 @@ class PyName(object): def load_module(path, name): + if not name: + name = os.path.basename(path) + name = name.rpartition('.')[0] # cut file type (normally .so) + sys_path = get_sys_path() if path: sys_path.insert(0, path) From f755e615c96a8d3f9d40adbead263813e8beba86 Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Thu, 9 Jan 2014 17:06:02 +0100 Subject: [PATCH 13/52] magic_method -> magic_function --- jedi/api/__init__.py | 2 +- jedi/evaluate/__init__.py | 2 +- jedi/evaluate/representation.py | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/jedi/api/__init__.py b/jedi/api/__init__.py index ce50e546..d8451662 100644 --- a/jedi/api/__init__.py +++ b/jedi/api/__init__.py @@ -192,7 +192,7 @@ class Script(object): debug.dbg('possible scopes', scopes) for s in scopes: if s.isinstance(er.Function): - names = s.get_magic_method_names() + names = s.get_magic_function_names() else: if isinstance(s, imports.ImportPath): under = like + self._user_context.get_path_after_cursor() diff --git a/jedi/evaluate/__init__.py b/jedi/evaluate/__init__.py index bd0690b4..904256c3 100644 --- a/jedi/evaluate/__init__.py +++ b/jedi/evaluate/__init__.py @@ -372,7 +372,7 @@ class Evaluator(object): else: # The function must not be decorated with something else. if type.isinstance(er.Function): - type = type.get_magic_method_scope() + type = type.get_magic_function_scope() else: # This is the typical lookup while chaining things. if filter_private_variable(type, scope, current): diff --git a/jedi/evaluate/representation.py b/jedi/evaluate/representation.py index 5a9d39e3..fa99f538 100644 --- a/jedi/evaluate/representation.py +++ b/jedi/evaluate/representation.py @@ -375,10 +375,10 @@ class Function(use_metaclass(CachedMetaClass, pr.IsScope)): return Function(self._evaluator, self.base_func, True) return decorated_func - def get_magic_method_names(self): + def get_magic_function_names(self): return compiled.magic_function_scope(self._evaluator).get_defined_names() - def get_magic_method_scope(self): + def get_magic_function_scope(self): return compiled.magic_function_scope(self._evaluator) def __getattr__(self, name): From dfd9a779c35a93573de7f6c75c7fd3268df9644f Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Thu, 9 Jan 2014 17:09:31 +0100 Subject: [PATCH 14/52] fix magic_function issues with compiled module --- jedi/evaluate/compiled.py | 1 + jedi/evaluate/representation.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/jedi/evaluate/compiled.py b/jedi/evaluate/compiled.py index c70d0b7a..78e5afd9 100644 --- a/jedi/evaluate/compiled.py +++ b/jedi/evaluate/compiled.py @@ -212,3 +212,4 @@ def _parse_function_doc(doc): builtin = PyObject(_builtins) +magic_function_class = PyObject(type(load_module), parent=builtin) diff --git a/jedi/evaluate/representation.py b/jedi/evaluate/representation.py index fa99f538..d714548f 100644 --- a/jedi/evaluate/representation.py +++ b/jedi/evaluate/representation.py @@ -376,10 +376,10 @@ class Function(use_metaclass(CachedMetaClass, pr.IsScope)): return decorated_func def get_magic_function_names(self): - return compiled.magic_function_scope(self._evaluator).get_defined_names() + return compiled.magic_function_class.get_defined_names() def get_magic_function_scope(self): - return compiled.magic_function_scope(self._evaluator) + return compiled.magic_function_class.get_defined_names() def __getattr__(self, name): return getattr(self.base_func, name) From 0234c1429bf05112e64a890ef64ad9162d3d3a66 Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Thu, 9 Jan 2014 17:54:40 +0100 Subject: [PATCH 15/52] fix a few more tracebacks --- jedi/evaluate/__init__.py | 30 +++++++++++++++--------------- jedi/evaluate/representation.py | 2 +- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/jedi/evaluate/__init__.py b/jedi/evaluate/__init__.py index 904256c3..fd764f77 100644 --- a/jedi/evaluate/__init__.py +++ b/jedi/evaluate/__init__.py @@ -331,8 +331,8 @@ class Evaluator(object): results_new = [] iter_paths = itertools.tee(path, len(types)) - for i, type in enumerate(types): - fp = self._follow_path(iter_paths[i], type, call_scope, position=position) + for i, typ in enumerate(types): + fp = self._follow_path(iter_paths[i], typ, call_scope, position=position) if fp is not None: results_new += fp else: @@ -340,7 +340,7 @@ class Evaluator(object): return types return results_new - def _follow_path(self, path, type, scope, position=None): + def _follow_path(self, path, typ, scope, position=None): """ Uses a generator and tries to complete the path, e.g.:: @@ -354,31 +354,31 @@ class Evaluator(object): current = next(path) except StopIteration: return None - debug.dbg('_follow_path: %s in scope %s' % (current, type)) + debug.dbg('_follow_path: %s in scope %s' % (current, typ)) result = [] if isinstance(current, pr.Array): # This must be an execution, either () or []. if current.type == pr.Array.LIST: - if hasattr(type, 'get_index_types'): - result = type.get_index_types(current) + if hasattr(typ, 'get_index_types'): + result = typ.get_index_types(current) elif current.type not in [pr.Array.DICT]: # Scope must be a class or func - make an instance or execution. - debug.dbg('exe', type) - result = self.execute(type, current) + debug.dbg('exe', typ) + result = self.execute(typ, current) else: # Curly braces are not allowed, because they make no sense. - debug.warning('strange function call with {}', current, type) + debug.warning('strange function call with {}', current, typ) else: # The function must not be decorated with something else. - if type.isinstance(er.Function): - type = type.get_magic_function_scope() + if typ.isinstance(er.Function): + typ = typ.get_magic_function_scope() else: # This is the typical lookup while chaining things. - if filter_private_variable(type, scope, current): + if filter_private_variable(typ, scope, current): return [] - result = imports.strip_imports(self, self.find_types(type, current, - position=position)) + types = self.find_types(typ, current, position=position) + result = imports.strip_imports(self, types) return self.follow_path(path, set(result), scope, position=position) def execute(self, obj, params=(), evaluate_generator=False): @@ -391,7 +391,7 @@ class Evaluator(object): pass if isinstance(obj, compiled.PyObject): - return obj.execute(params) + return list(obj.execute(params)) elif obj.isinstance(er.Class): # There maybe executions of executions. return [er.Instance(self, obj, params)] diff --git a/jedi/evaluate/representation.py b/jedi/evaluate/representation.py index d714548f..bc834834 100644 --- a/jedi/evaluate/representation.py +++ b/jedi/evaluate/representation.py @@ -379,7 +379,7 @@ class Function(use_metaclass(CachedMetaClass, pr.IsScope)): return compiled.magic_function_class.get_defined_names() def get_magic_function_scope(self): - return compiled.magic_function_class.get_defined_names() + return compiled.magic_function_class def __getattr__(self, name): return getattr(self.base_func, name) From 9056dc1b9b82f9c7a4a1848af32040aba6a8ade1 Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Thu, 9 Jan 2014 19:55:00 +0100 Subject: [PATCH 16/52] fixed some array indexing --- jedi/evaluate/__init__.py | 14 +++++--------- jedi/evaluate/compiled.py | 7 ++++++- jedi/evaluate/iterable.py | 16 +++++----------- jedi/parser/representation.py | 3 --- 4 files changed, 16 insertions(+), 24 deletions(-) diff --git a/jedi/evaluate/__init__.py b/jedi/evaluate/__init__.py index fd764f77..f3ccbf16 100644 --- a/jedi/evaluate/__init__.py +++ b/jedi/evaluate/__init__.py @@ -307,16 +307,12 @@ class Evaluator(object): else: if isinstance(current, pr.NamePart): # This is the first global lookup. - scopes = self.find_types(scope, current, position=position, - search_global=True) + types = self.find_types(scope, current, position=position, + search_global=True) else: # for pr.Literal - scopes = self.find_types(compiled.builtin, current.type_as_string()) - # Make instances of those number/string objects. - scopes = itertools.chain.from_iterable( - self.execute(s, (current.value,)) for s in scopes - ) - types = imports.strip_imports(self, scopes) + types = [compiled.create(current.value)] + types = imports.strip_imports(self, types) return self.follow_path(path, types, scope, position=position) @@ -412,7 +408,7 @@ class Evaluator(object): else: stmts = er.FunctionExecution(self, obj, params).get_return_types(evaluate_generator) - debug.dbg('execute: %s in %s' % (stmts, self)) + debug.dbg('execute: %s in %s' % (stmts, obj)) return imports.strip_imports(self, stmts) def goto(self, stmt, call_path=None): diff --git a/jedi/evaluate/compiled.py b/jedi/evaluate/compiled.py index 78e5afd9..d2d54e5a 100644 --- a/jedi/evaluate/compiled.py +++ b/jedi/evaluate/compiled.py @@ -68,7 +68,8 @@ class PyObject(Base): def execute(self, params): t = self.type() if t == 'class': - yield PyObject(self.obj, self.parent, True) + if not self.instantiated: + yield PyObject(self.obj, self.parent, True) elif t == 'def': for name in self._parse_function_doc()[1].split(): try: @@ -213,3 +214,7 @@ def _parse_function_doc(doc): builtin = PyObject(_builtins) magic_function_class = PyObject(type(load_module), parent=builtin) + + +def create(obj): + return PyObject(obj, builtin) diff --git a/jedi/evaluate/iterable.py b/jedi/evaluate/iterable.py index 4fab6c1d..985b08b5 100644 --- a/jedi/evaluate/iterable.py +++ b/jedi/evaluate/iterable.py @@ -3,7 +3,7 @@ import itertools from jedi import common from jedi import debug from jedi import settings -from jedi._compatibility import use_metaclass, is_py3k +from jedi._compatibility import use_metaclass, is_py3k, unicode from jedi.parser import representation as pr from jedi.evaluate import compiled from jedi.evaluate import helpers @@ -78,16 +78,10 @@ class Array(use_metaclass(CachedMetaClass, pr.Base)): # This is indexing only one element, with a fixed index number, # otherwise it just ignores the index (e.g. [1+1]). index = index_possibilities[0] - - from jedi.evaluate.representation import Instance - if isinstance(index, Instance) \ - and str(index.name) in ['int', 'str'] \ - and len(index.var_args) == 1: - # TODO this is just very hackish and a lot of use cases are - # being ignored - with common.ignored(KeyError, IndexError, - UnboundLocalError, TypeError): - return self.get_exact_index_types(index.var_args[0]) + if isinstance(index, compiled.PyObject) \ + and isinstance(index.obj, (int, str, unicode)): + with common.ignored(KeyError, IndexError, TypeError): + return self.get_exact_index_types(index.obj) result = list(self._follow_values(self._array.values)) result += check_array_additions(self._evaluator, self) diff --git a/jedi/parser/representation.py b/jedi/parser/representation.py index b9feea5e..10e9882b 100644 --- a/jedi/parser/representation.py +++ b/jedi/parser/representation.py @@ -1239,9 +1239,6 @@ class Literal(StatementElement): def get_code(self): return self.literal + super(Literal, self).get_code() - def type_as_string(self): - return type(self.value).__name__ - def __repr__(self): if is_py3k: s = self.literal From 59b379ccc5fca3199a4f90e498d8eeff54a47002 Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Fri, 10 Jan 2014 00:52:39 +0100 Subject: [PATCH 17/52] fixed most function issues --- jedi/evaluate/compiled.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jedi/evaluate/compiled.py b/jedi/evaluate/compiled.py index d2d54e5a..6c7ecab8 100644 --- a/jedi/evaluate/compiled.py +++ b/jedi/evaluate/compiled.py @@ -63,7 +63,7 @@ class PyObject(Base): @property def name(self): # might not exist sometimes (raises AttributeError) - return self.obj.__name__ + return self._cls().obj.__name__ def execute(self, params): t = self.type() From b1409c8f7489db6a6bdacc6f3f7cd8b4777fe4f9 Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Fri, 10 Jan 2014 02:03:02 +0100 Subject: [PATCH 18/52] fix a name glitch in keywords --- jedi/keywords.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/jedi/keywords.py b/jedi/keywords.py index df26202a..7fea1b31 100644 --- a/jedi/keywords.py +++ b/jedi/keywords.py @@ -2,7 +2,6 @@ import pydoc import keyword from jedi._compatibility import is_py3k -from jedi.parser import representation as pr from jedi import common from jedi.evaluate import compiled @@ -30,8 +29,7 @@ def keyword_names(*args, **kwargs): kwds = [] for k in keywords(*args, **kwargs): start = k.start_pos - end = start[0], start[1] + len(k.name) - kwds.append(pr.Name(k.parent, [(k.name, start)], start, end, k)) + kwds.append(KeywordName(k, k.name, start)) return kwds @@ -39,6 +37,17 @@ def get_operator(string, pos): return Keyword(string, pos) +class KeywordName(object): + def __init__(self, parent, name, start_pos): + self.parent = parent + self.names = [name] + self.start_pos = start_pos + + @property + def end_pos(self): + return self.start_pos[0], self.start_pos[1] + len(self.name) + + class Keyword(object): def __init__(self, name, pos): self.name = name From f868668f0e521b98ab1ac301bbb82057addcd1f7 Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Fri, 10 Jan 2014 13:36:29 +0100 Subject: [PATCH 19/52] trying to fix the getattr mess with compiled --- jedi/evaluate/__init__.py | 5 ++++- jedi/evaluate/compiled.py | 13 +++++++++++-- jedi/evaluate/finder.py | 10 +++++----- jedi/evaluate/param.py | 3 +-- jedi/evaluate/stdlib.py | 15 ++++++++------- test/completion/classes.py | 2 +- 6 files changed, 30 insertions(+), 18 deletions(-) diff --git a/jedi/evaluate/__init__.py b/jedi/evaluate/__init__.py index f3ccbf16..60d74ef3 100644 --- a/jedi/evaluate/__init__.py +++ b/jedi/evaluate/__init__.py @@ -258,7 +258,10 @@ class Evaluator(object): call.stmt.parent = loop result += self.eval_statement(call.stmt) else: - if isinstance(call, pr.Lambda): + if isinstance(call, compiled.PyName): + print call, call.parent + result.append(call.parent) + elif isinstance(call, pr.Lambda): result.append(er.Function(self, call)) # With things like params, these can also be functions... elif isinstance(call, pr.Base) and call.isinstance( diff --git a/jedi/evaluate/compiled.py b/jedi/evaluate/compiled.py index 6c7ecab8..a7c4f9a9 100644 --- a/jedi/evaluate/compiled.py +++ b/jedi/evaluate/compiled.py @@ -51,7 +51,9 @@ class PyObject(Base): @underscore_memoization def _cls(self): # Ensures that a PyObject is returned that is not an instance (like list) - if not (inspect.isclass(self.obj) or inspect.ismodule(self.obj)): + if not (inspect.isclass(self.obj) or inspect.ismodule(self.obj) + or inspect.isbuiltin(self.obj) or inspect.ismethod(self.obj) + or inspect.ismethoddescriptor(self.obj)): return PyObject(self.obj.__class__, self.parent, True) return self @@ -89,9 +91,12 @@ class PyName(object): self._obj = obj self._name = name self.start_pos = 0, 0 # an illegal start_pos, to make sorting easy. + #if not type(name) is str: + # print obj, name + # raise NotImplementedError() def __repr__(self): - return '<%s: %s.%s>' % (type(self).__name__, self._obj.obj, self._name) + return '<%s: (%s).%s>' % (type(self).__name__, repr(self._obj.obj), self._name) @property @underscore_memoization @@ -218,3 +223,7 @@ magic_function_class = PyObject(type(load_module), parent=builtin) def create(obj): return PyObject(obj, builtin) + + +def name_from_string(string): + return PyName(create(string), string) diff --git a/jedi/evaluate/finder.py b/jedi/evaluate/finder.py index 96e15d9f..adf90e3b 100644 --- a/jedi/evaluate/finder.py +++ b/jedi/evaluate/finder.py @@ -71,9 +71,10 @@ class NameFinder(object): if not result and isinstance(self.scope, er.Instance): # __getattr__ / __getattribute__ for r in self._check_getattr(self.scope): - new_name = copy.copy(r.name) - new_name.parent = r - result.append(new_name) + if not isinstance(r, compiled.PyObject): + new_name = copy.copy(r.name) + new_name.parent = r + result.append(new_name) debug.dbg('sfn filter "%s" in (%s-%s): %s@%s' % (self.name_str, self.scope, nscope, u(result), self.position)) @@ -82,9 +83,8 @@ class NameFinder(object): def _check_getattr(self, inst): """Checks for both __getattr__ and __getattribute__ methods""" result = [] - module = compiled.builtin # str is important to lose the NamePart! - name = pr.String(module, "'%s'" % self.name_str, (0, 0), (0, 0), inst) + name = compiled.name_from_string(self.name_str) with common.ignored(KeyError): result = inst.execute_subscope_by_name('__getattr__', [name]) if not result: diff --git a/jedi/evaluate/param.py b/jedi/evaluate/param.py index eb4f59eb..c363ed61 100644 --- a/jedi/evaluate/param.py +++ b/jedi/evaluate/param.py @@ -2,7 +2,6 @@ import copy from jedi.parser import representation as pr from jedi.evaluate import iterable -from jedi.evaluate import compiled from jedi.evaluate import common @@ -140,7 +139,7 @@ def _var_args_iterator(evaluator, var_args): continue old = stmt # generate a statement if it's not already one. - stmt = pr.Statement(compiled.builtin, [], (0, 0), None) + stmt = pr.Statement(_FakeSubModule, [], (0, 0), None) stmt._expression_list = [old] # *args diff --git a/jedi/evaluate/stdlib.py b/jedi/evaluate/stdlib.py index 4d821579..4f1feca7 100644 --- a/jedi/evaluate/stdlib.py +++ b/jedi/evaluate/stdlib.py @@ -3,6 +3,7 @@ Implementations of standard library functions, because it's not possible to understand them with Jedi. """ +from jedi._compatibility import unicode from jedi.evaluate import compiled from jedi.evaluate import representation as er from jedi.evaluate import iterable @@ -44,18 +45,18 @@ def builtins_getattr(evaluator, obj, params): objects = _follow_param(evaluator, params, 0) names = _follow_param(evaluator, params, 1) for obj in objects: - if not isinstance(obj, (er.Instance, er.Class, pr.Module)): + if not isinstance(obj, (er.Instance, er.Class, pr.Module, compiled.PyObject)): debug.warning('getattr called without instance') continue - for arr_name in names: - if not isinstance(arr_name, er.Instance): + for name in names: + s = unicode, str + print name + if isinstance(name, compiled.PyObject) and isinstance(name.obj, s): + stmts += evaluator.follow_path(iter([name.obj]), [obj], obj) + else: debug.warning('getattr called without str') continue - if len(arr_name.var_args) != 1: - debug.warning('jedi getattr is too simple') - key = arr_name.var_args[0] - stmts += evaluator.follow_path(iter([key]), [obj], obj) return stmts diff --git a/test/completion/classes.py b/test/completion/classes.py index 44967f6a..9ae49633 100644 --- a/test/completion/classes.py +++ b/test/completion/classes.py @@ -339,7 +339,7 @@ class Wrapper2(): return getattr(Base(), name) #? int() -Wrapper(Base()).ret(3) +Wrapper(Base()).ret #? int() Wrapper2(Base()).ret(3) From 14c9ed88ca4f055499016b0b18b8d20d052fb672 Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Fri, 10 Jan 2014 15:14:55 +0100 Subject: [PATCH 20/52] fixed the getattr/__getattr__ stuff --- jedi/evaluate/__init__.py | 7 +++---- jedi/evaluate/compiled.py | 7 ------- jedi/evaluate/finder.py | 2 +- jedi/evaluate/stdlib.py | 1 - test/completion/classes.py | 2 +- 5 files changed, 5 insertions(+), 14 deletions(-) diff --git a/jedi/evaluate/__init__.py b/jedi/evaluate/__init__.py index 60d74ef3..81e0e5af 100644 --- a/jedi/evaluate/__init__.py +++ b/jedi/evaluate/__init__.py @@ -258,16 +258,15 @@ class Evaluator(object): call.stmt.parent = loop result += self.eval_statement(call.stmt) else: - if isinstance(call, compiled.PyName): - print call, call.parent - result.append(call.parent) - elif isinstance(call, pr.Lambda): + if isinstance(call, pr.Lambda): result.append(er.Function(self, call)) # With things like params, these can also be functions... elif isinstance(call, pr.Base) and call.isinstance( er.Function, er.Class, er.Instance, iterable.ArrayInstance): result.append(call) # The string tokens are just operations (+, -, etc.) + elif isinstance(call, compiled.PyObject): + result.append(call) elif not isinstance(call, (str, unicode)): if isinstance(call, pr.Call) and str(call.name) == 'if': # Ternary operators. diff --git a/jedi/evaluate/compiled.py b/jedi/evaluate/compiled.py index a7c4f9a9..82083fee 100644 --- a/jedi/evaluate/compiled.py +++ b/jedi/evaluate/compiled.py @@ -91,9 +91,6 @@ class PyName(object): self._obj = obj self._name = name self.start_pos = 0, 0 # an illegal start_pos, to make sorting easy. - #if not type(name) is str: - # print obj, name - # raise NotImplementedError() def __repr__(self): return '<%s: (%s).%s>' % (type(self).__name__, repr(self._obj.obj), self._name) @@ -223,7 +220,3 @@ magic_function_class = PyObject(type(load_module), parent=builtin) def create(obj): return PyObject(obj, builtin) - - -def name_from_string(string): - return PyName(create(string), string) diff --git a/jedi/evaluate/finder.py b/jedi/evaluate/finder.py index adf90e3b..007e202c 100644 --- a/jedi/evaluate/finder.py +++ b/jedi/evaluate/finder.py @@ -84,7 +84,7 @@ class NameFinder(object): """Checks for both __getattr__ and __getattribute__ methods""" result = [] # str is important to lose the NamePart! - name = compiled.name_from_string(self.name_str) + name = compiled.create(str(self.name_str)) with common.ignored(KeyError): result = inst.execute_subscope_by_name('__getattr__', [name]) if not result: diff --git a/jedi/evaluate/stdlib.py b/jedi/evaluate/stdlib.py index 4f1feca7..3d31f30d 100644 --- a/jedi/evaluate/stdlib.py +++ b/jedi/evaluate/stdlib.py @@ -51,7 +51,6 @@ def builtins_getattr(evaluator, obj, params): for name in names: s = unicode, str - print name if isinstance(name, compiled.PyObject) and isinstance(name.obj, s): stmts += evaluator.follow_path(iter([name.obj]), [obj], obj) else: diff --git a/test/completion/classes.py b/test/completion/classes.py index 9ae49633..44967f6a 100644 --- a/test/completion/classes.py +++ b/test/completion/classes.py @@ -339,7 +339,7 @@ class Wrapper2(): return getattr(Base(), name) #? int() -Wrapper(Base()).ret +Wrapper(Base()).ret(3) #? int() Wrapper2(Base()).ret(3) From 8854206f2a2bde84b0e99dcd0912bf33d1f80530 Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Fri, 10 Jan 2014 16:36:53 +0100 Subject: [PATCH 21/52] created a module for compiled --- jedi/evaluate/{compiled.py => compiled/__init__.py} | 0 setup.py | 4 ++-- 2 files changed, 2 insertions(+), 2 deletions(-) rename jedi/evaluate/{compiled.py => compiled/__init__.py} (100%) diff --git a/jedi/evaluate/compiled.py b/jedi/evaluate/compiled/__init__.py similarity index 100% rename from jedi/evaluate/compiled.py rename to jedi/evaluate/compiled/__init__.py diff --git a/setup.py b/setup.py index 2fb4f277..6b6f8963 100755 --- a/setup.py +++ b/setup.py @@ -26,8 +26,8 @@ setup(name='jedi', license='MIT', keywords='python completion refactoring vim', long_description=readme, - packages=['jedi', 'jedi.parser', 'jedi.evaluate', 'jedi.api'], - package_data={'jedi': ['evlaluate/evaluate/mixin/*.pym']}, + packages=['jedi', 'jedi.parser', 'jedi.evaluate', 'jedi.evaluate.compiled', 'jedi.api'], + package_data={'jedi': ['evlaluate/evaluate/compiled/fake/*.pym']}, platforms=['any'], classifiers=[ 'Development Status :: 4 - Beta', From 400b0a4aa7d09749dc71fb0d2bd7e2767b445ade Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Fri, 10 Jan 2014 16:38:13 +0100 Subject: [PATCH 22/52] move mixin to compiled/fake to structure it better --- jedi/evaluate/{mixin => compiled/fake}/_functools.pym | 0 jedi/evaluate/{mixin => compiled/fake}/_io.pym | 0 jedi/evaluate/{mixin => compiled/fake}/_sqlite3.pym | 0 jedi/evaluate/{mixin => compiled/fake}/_sre.pym | 0 jedi/evaluate/{mixin => compiled/fake}/_weakref.pym | 0 jedi/evaluate/{mixin => compiled/fake}/builtins.pym | 0 jedi/evaluate/{mixin => compiled/fake}/datetime.pym | 0 jedi/evaluate/{mixin => compiled/fake}/posix.pym | 0 8 files changed, 0 insertions(+), 0 deletions(-) rename jedi/evaluate/{mixin => compiled/fake}/_functools.pym (100%) rename jedi/evaluate/{mixin => compiled/fake}/_io.pym (100%) rename jedi/evaluate/{mixin => compiled/fake}/_sqlite3.pym (100%) rename jedi/evaluate/{mixin => compiled/fake}/_sre.pym (100%) rename jedi/evaluate/{mixin => compiled/fake}/_weakref.pym (100%) rename jedi/evaluate/{mixin => compiled/fake}/builtins.pym (100%) rename jedi/evaluate/{mixin => compiled/fake}/datetime.pym (100%) rename jedi/evaluate/{mixin => compiled/fake}/posix.pym (100%) diff --git a/jedi/evaluate/mixin/_functools.pym b/jedi/evaluate/compiled/fake/_functools.pym similarity index 100% rename from jedi/evaluate/mixin/_functools.pym rename to jedi/evaluate/compiled/fake/_functools.pym diff --git a/jedi/evaluate/mixin/_io.pym b/jedi/evaluate/compiled/fake/_io.pym similarity index 100% rename from jedi/evaluate/mixin/_io.pym rename to jedi/evaluate/compiled/fake/_io.pym diff --git a/jedi/evaluate/mixin/_sqlite3.pym b/jedi/evaluate/compiled/fake/_sqlite3.pym similarity index 100% rename from jedi/evaluate/mixin/_sqlite3.pym rename to jedi/evaluate/compiled/fake/_sqlite3.pym diff --git a/jedi/evaluate/mixin/_sre.pym b/jedi/evaluate/compiled/fake/_sre.pym similarity index 100% rename from jedi/evaluate/mixin/_sre.pym rename to jedi/evaluate/compiled/fake/_sre.pym diff --git a/jedi/evaluate/mixin/_weakref.pym b/jedi/evaluate/compiled/fake/_weakref.pym similarity index 100% rename from jedi/evaluate/mixin/_weakref.pym rename to jedi/evaluate/compiled/fake/_weakref.pym diff --git a/jedi/evaluate/mixin/builtins.pym b/jedi/evaluate/compiled/fake/builtins.pym similarity index 100% rename from jedi/evaluate/mixin/builtins.pym rename to jedi/evaluate/compiled/fake/builtins.pym diff --git a/jedi/evaluate/mixin/datetime.pym b/jedi/evaluate/compiled/fake/datetime.pym similarity index 100% rename from jedi/evaluate/mixin/datetime.pym rename to jedi/evaluate/compiled/fake/datetime.pym diff --git a/jedi/evaluate/mixin/posix.pym b/jedi/evaluate/compiled/fake/posix.pym similarity index 100% rename from jedi/evaluate/mixin/posix.pym rename to jedi/evaluate/compiled/fake/posix.pym From 78cc015b9d03538f790b241826e29a910f58b76d Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Fri, 10 Jan 2014 22:58:49 +0100 Subject: [PATCH 23/52] start introducing the compiled.fake module that fakes builtin code --- jedi/evaluate/compiled/__init__.py | 27 ++++++-- jedi/evaluate/compiled/fake.py | 106 +++++++++++++++++++++++++++++ 2 files changed, 128 insertions(+), 5 deletions(-) create mode 100644 jedi/evaluate/compiled/fake.py diff --git a/jedi/evaluate/compiled/__init__.py b/jedi/evaluate/compiled/__init__.py index 82083fee..19f6e404 100644 --- a/jedi/evaluate/compiled/__init__.py +++ b/jedi/evaluate/compiled/__init__.py @@ -11,6 +11,7 @@ from jedi import debug from jedi.parser.representation import Base from jedi.cache import underscore_memoization from jedi.evaluate.sys_path import get_sys_path +from . import fake # TODO @@ -51,9 +52,7 @@ class PyObject(Base): @underscore_memoization def _cls(self): # Ensures that a PyObject is returned that is not an instance (like list) - if not (inspect.isclass(self.obj) or inspect.ismodule(self.obj) - or inspect.isbuiltin(self.obj) or inspect.ismethod(self.obj) - or inspect.ismethoddescriptor(self.obj)): + if fake.is_class_instance(self.obj): return PyObject(self.obj.__class__, self.parent, True) return self @@ -218,5 +217,23 @@ builtin = PyObject(_builtins) magic_function_class = PyObject(type(load_module), parent=builtin) -def create(obj): - return PyObject(obj, builtin) +def create(obj, parent=builtin, instantiated=False, module=None): + if module is None: + if not inspect.ismodule(obj): + module = obj.__class__ if fake.is_class_instance(obj) else obj + if not (inspect.isbuiltin(module) or inspect.isclass(module)): + module = obj.__objclass__ + try: + imp_plz = obj.__module__ + except AttributeError: + # Unfortunately in some cases like `int` there's no __module__ + module = builtin + else: + module = PyObject(__import__(imp_plz)) + + faked = fake.get_faked(module.obj, obj) + if faked is not None: + faked.parent = parent + return faked + + return PyObject(obj, parent, instantiated) diff --git a/jedi/evaluate/compiled/fake.py b/jedi/evaluate/compiled/fake.py new file mode 100644 index 00000000..f2a8e442 --- /dev/null +++ b/jedi/evaluate/compiled/fake.py @@ -0,0 +1,106 @@ +""" +Loads functions that are mixed in to the standard library. E.g. builtins are +written in C (binaries), but my autocompletion only understands Python code. By +mixing in Python code, the autocompletion should work much better for builtins. +""" + +import re +import os +import inspect + +from jedi._compatibility import is_py3k +from jedi.parser import Parser + +modules = {} + + +def _load_fakes(module_name): + regex = r'^(def|class)\s+([\w\d]+)' + + def process_code(code, depth=0): + funcs = {} + matches = list(re.finditer(regex, code, re.MULTILINE)) + positions = [m.start() for m in matches] + for i, pos in enumerate(positions): + try: + code_block = code[pos:positions[i + 1]] + except IndexError: + code_block = code[pos:len(code)] + structure_name = matches[i].group(1) + name = matches[i].group(2) + if structure_name == 'def': + funcs[name] = code_block + elif structure_name == 'class': + if depth > 0: + raise NotImplementedError() + + # remove class line + c = re.sub(r'^[^\n]+', '', code_block) + # remove whitespace + c = re.compile(r'^[ ]{4}', re.MULTILINE).sub('', c) + + funcs[name] = process_code(c) + else: + raise NotImplementedError() + return funcs + + # sometimes there are stupid endings like `_sqlite3.cpython-32mu` + module_name = re.sub(r'\..*', '', module_name) + + if module_name == '__builtin__' and not is_py3k: + module_name = 'builtins' + path = os.path.dirname(os.path.abspath(__file__)) + try: + with open(os.path.join(path, 'mixin', module_name) + '.pym') as f: + s = f.read() + except IOError: + return {} + else: + mixin_dct = process_code(s) + if is_py3k and module_name == 'builtins': + # in the case of Py3k xrange is now range + mixin_dct['range'] = mixin_dct['xrange'] + return mixin_dct + + +def _load_module(module): + module_name = module.__name__ + try: + return modules[module_name] + except KeyError: + path = os.path.dirname(os.path.abspath(__file__)) + try: + with open(os.path.join(path, 'fake', module_name) + '.pym') as f: + source = f.read() + except IOError: + return {} + module = Parser(source).module + modules[module_name] = module + return module + + +def get_faked(module, obj): + def from_scope(scope, obj): + for s in scope.subscopes: + if s.name == obj.name: + return s + + mod = _load_module(module) + + # Having the module as a `parser.representation.module`, we need to scan + # for methods. + if is_class_instance(obj): + obj = obj.__class__ + if inspect.isbuiltin(obj): + return from_scope(mod, obj) + elif not inspect.isclass(obj): + # object is a method or descriptor + cls = obj.__objclass__ + return from_scope(from_scope(mod, cls), obj) + + +def is_class_instance(obj): + """Like inspect.* methods.""" + return not (inspect.isclass(obj) or inspect.ismodule(obj) + or inspect.isbuiltin(obj) or inspect.ismethod(obj) + or inspect.ismethoddescriptor(obj)) From 01c03966a7d154ba8a1f4d40c031b273b9e76e35 Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Fri, 10 Jan 2014 23:35:53 +0100 Subject: [PATCH 24/52] make first faked compiled modules work --- jedi/evaluate/compiled/__init__.py | 4 ++-- jedi/evaluate/compiled/fake.py | 6 ++++-- test/test_compiled.py | 5 +++++ 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/jedi/evaluate/compiled/__init__.py b/jedi/evaluate/compiled/__init__.py index 19f6e404..b3538846 100644 --- a/jedi/evaluate/compiled/__init__.py +++ b/jedi/evaluate/compiled/__init__.py @@ -74,7 +74,7 @@ class PyObject(Base): elif t == 'def': for name in self._parse_function_doc()[1].split(): try: - yield PyObject(getattr(_builtins, name), builtin, True) + yield create(getattr(_builtins, name), builtin, True) except AttributeError: pass @@ -99,7 +99,7 @@ class PyName(object): def parent(self): try: # this has a builtin_function_or_method - return PyObject(getattr(self._obj.obj, self._name), self._obj) + return create(getattr(self._obj.obj, self._name), self._obj) except AttributeError: # happens e.g. in properties of # PyQt4.QtGui.QStyleOptionComboBox.currentText diff --git a/jedi/evaluate/compiled/fake.py b/jedi/evaluate/compiled/fake.py index f2a8e442..61504159 100644 --- a/jedi/evaluate/compiled/fake.py +++ b/jedi/evaluate/compiled/fake.py @@ -65,6 +65,8 @@ def _load_fakes(module_name): def _load_module(module): module_name = module.__name__ + if module_name == '__builtin__' and not is_py3k: + module_name = 'builtins' try: return modules[module_name] except KeyError: @@ -74,7 +76,7 @@ def _load_module(module): source = f.read() except IOError: return {} - module = Parser(source).module + module = Parser(source, module_name).module modules[module_name] = module return module @@ -82,7 +84,7 @@ def _load_module(module): def get_faked(module, obj): def from_scope(scope, obj): for s in scope.subscopes: - if s.name == obj.name: + if str(s.name) == obj.__name__: return s mod = _load_module(module) diff --git a/test/test_compiled.py b/test/test_compiled.py index 930c27e4..203c76c3 100644 --- a/test/test_compiled.py +++ b/test/test_compiled.py @@ -1,4 +1,5 @@ from jedi._compatibility import builtins +from jedi.parser.representation import Function from jedi.evaluate import compiled from jedi.evaluate import Evaluator @@ -13,3 +14,7 @@ def test_simple(): assert len(objs) == 1 assert objs[0].obj is str assert objs[0].instantiated is True + + +def test_fake_loading(): + assert isinstance(compiled.create(reversed), Function) From 32e39ef4cac5a08738a2048a4c6092411998dc22 Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Sat, 11 Jan 2014 00:26:53 +0100 Subject: [PATCH 25/52] fixing parents in compiled --- jedi/evaluate/compiled/__init__.py | 13 ++++++++----- test/test_compiled.py | 2 +- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/jedi/evaluate/compiled/__init__.py b/jedi/evaluate/compiled/__init__.py index b3538846..749a781b 100644 --- a/jedi/evaluate/compiled/__init__.py +++ b/jedi/evaluate/compiled/__init__.py @@ -26,12 +26,15 @@ class PyObject(Base): self.doc = inspect.getdoc(obj) # comply with the parser - self.get_parent_until = lambda *args, **kwargs: parent or self self.start_pos = 0, 0 def __repr__(self): return '<%s: %s>' % (type(self).__name__, self.obj) + def get_parent_until(self, *args, **kwargs): + # compiled modules only use functions and classes/methods (2 levels) + return getattr(self.parent, 'parent', self.parent) or self + @underscore_memoization def _parse_function_doc(self): if self.doc is None: @@ -99,7 +102,8 @@ class PyName(object): def parent(self): try: # this has a builtin_function_or_method - return create(getattr(self._obj.obj, self._name), self._obj) + return create(getattr(self._obj.obj, self._name), self._obj, + module=self._obj.get_parent_until()) except AttributeError: # happens e.g. in properties of # PyQt4.QtGui.QStyleOptionComboBox.currentText @@ -160,7 +164,6 @@ def _parse_function_doc(doc): TODO docstrings like utime(path, (atime, mtime)) and a(b [, b]) -> None TODO docstrings like 'tuple of integers' """ - # parse round parentheses: def func(a, (b,c)) try: count = 0 @@ -218,8 +221,8 @@ magic_function_class = PyObject(type(load_module), parent=builtin) def create(obj, parent=builtin, instantiated=False, module=None): - if module is None: - if not inspect.ismodule(obj): + if not inspect.ismodule(obj): + if module is None: module = obj.__class__ if fake.is_class_instance(obj) else obj if not (inspect.isbuiltin(module) or inspect.isclass(module)): module = obj.__objclass__ diff --git a/test/test_compiled.py b/test/test_compiled.py index 203c76c3..287549ee 100644 --- a/test/test_compiled.py +++ b/test/test_compiled.py @@ -17,4 +17,4 @@ def test_simple(): def test_fake_loading(): - assert isinstance(compiled.create(reversed), Function) + assert isinstance(compiled.create(next), Function) From 8337f778862c534ced2d3cf812feb8e7e58dfbb4 Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Sat, 11 Jan 2014 01:19:09 +0100 Subject: [PATCH 26/52] a few other small changes before changing compiled Instance execution to the representation --- jedi/evaluate/__init__.py | 11 +++++------ jedi/evaluate/compiled/__init__.py | 12 +++++++++--- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/jedi/evaluate/__init__.py b/jedi/evaluate/__init__.py index 81e0e5af..c9791d86 100644 --- a/jedi/evaluate/__init__.py +++ b/jedi/evaluate/__init__.py @@ -195,13 +195,12 @@ class Evaluator(object): @recursion.recursion_decorator def eval_statement(self, stmt, seek_name=None): """ - The starting point of the completion. A statement always owns a call list, - which are the calls, that a statement does. - In case multiple names are defined in the statement, `seek_name` returns - the result for this name. + The starting point of the completion. A statement always owns a call + list, which are the calls, that a statement does. In case multiple + names are defined in the statement, `seek_name` returns the result for + this name. :param stmt: A `pr.Statement`. - :param seek_name: A string. """ debug.dbg('eval_statement %s (%s)' % (stmt, seek_name)) expression_list = stmt.expression_list() @@ -389,7 +388,7 @@ class Evaluator(object): pass if isinstance(obj, compiled.PyObject): - return list(obj.execute(params)) + return list(obj.execute(self, params)) elif obj.isinstance(er.Class): # There maybe executions of executions. return [er.Instance(self, obj, params)] diff --git a/jedi/evaluate/compiled/__init__.py b/jedi/evaluate/compiled/__init__.py index 749a781b..112a6141 100644 --- a/jedi/evaluate/compiled/__init__.py +++ b/jedi/evaluate/compiled/__init__.py @@ -69,7 +69,7 @@ class PyObject(Base): # might not exist sometimes (raises AttributeError) return self._cls().obj.__name__ - def execute(self, params): + def execute(self, evaluator, params): t = self.type() if t == 'class': if not self.instantiated: @@ -77,9 +77,15 @@ class PyObject(Base): elif t == 'def': for name in self._parse_function_doc()[1].split(): try: - yield create(getattr(_builtins, name), builtin, True) + bltn_obj = create(getattr(_builtins, name), builtin, module=builtin) except AttributeError: - pass + continue + else: + if isinstance(bltn_obj, PyObject): + yield bltn_obj + else: + for result in evaluator.execute(bltn_obj, params): + yield result def get_self_attributes(self): return [] # Instance compatibility From 19fa320c88812444e5d95378999cfb5333172a20 Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Sat, 11 Jan 2014 01:48:59 +0100 Subject: [PATCH 27/52] a hopefully simple integration of PyObject into Instance --- jedi/evaluate/__init__.py | 9 ++++-- jedi/evaluate/compiled/__init__.py | 47 ++++++++++++++++++------------ jedi/evaluate/representation.py | 2 ++ 3 files changed, 36 insertions(+), 22 deletions(-) diff --git a/jedi/evaluate/__init__.py b/jedi/evaluate/__init__.py index c9791d86..978202a4 100644 --- a/jedi/evaluate/__init__.py +++ b/jedi/evaluate/__init__.py @@ -361,7 +361,6 @@ class Evaluator(object): result = typ.get_index_types(current) elif current.type not in [pr.Array.DICT]: # Scope must be a class or func - make an instance or execution. - debug.dbg('exe', typ) result = self.execute(typ, current) else: # Curly braces are not allowed, because they make no sense. @@ -382,13 +381,17 @@ class Evaluator(object): if obj.isinstance(er.Function): obj = obj.get_decorated_func() + debug.dbg('execute:', obj, params) try: return stdlib.execute(self, obj, params) except stdlib.NotInStdLib: pass if isinstance(obj, compiled.PyObject): - return list(obj.execute(self, params)) + if obj.is_executable_class(): + return [er.Instance(self, obj, params)] + else: + return list(obj.execute_function(self, params)) elif obj.isinstance(er.Class): # There maybe executions of executions. return [er.Instance(self, obj, params)] @@ -409,7 +412,7 @@ class Evaluator(object): else: stmts = er.FunctionExecution(self, obj, params).get_return_types(evaluate_generator) - debug.dbg('execute: %s in %s' % (stmts, obj)) + debug.dbg('execute result: %s in %s' % (stmts, obj)) return imports.strip_imports(self, stmts) def goto(self, stmt, call_path=None): diff --git a/jedi/evaluate/compiled/__init__.py b/jedi/evaluate/compiled/__init__.py index 112a6141..390716e2 100644 --- a/jedi/evaluate/compiled/__init__.py +++ b/jedi/evaluate/compiled/__init__.py @@ -19,15 +19,16 @@ from . import fake # if not hasattr(func, "__name__"): class PyObject(Base): + # comply with the parser + start_pos = 0, 0 + asserts = [] + def __init__(self, obj, parent=None, instantiated=False): self.obj = obj self.parent = parent self.instantiated = instantiated self.doc = inspect.getdoc(obj) - # comply with the parser - self.start_pos = 0, 0 - def __repr__(self): return '<%s: %s>' % (type(self).__name__, self.obj) @@ -52,6 +53,9 @@ class PyObject(Base): or inspect.ismethoddescriptor(cls): return 'def' + def is_executable_class(self): + return inspect.isclass(self.obj) + @underscore_memoization def _cls(self): # Ensures that a PyObject is returned that is not an instance (like list) @@ -64,28 +68,33 @@ class PyObject(Base): for name in dir(cls.obj): yield PyName(cls, name) + def instance_names(self): + # TODO REMOVE (temporary until the Instance method is removed) + return self.get_defined_names() + + def get_subscope_by_name(self, name): + if name in dir(self._cls().obj): + return PyName(self._cls, name).parent + else: + raise KeyError("CompiledObject doesn't have an attribute '%s'." % name) + @property def name(self): # might not exist sometimes (raises AttributeError) return self._cls().obj.__name__ - def execute(self, evaluator, params): - t = self.type() - if t == 'class': - if not self.instantiated: - yield PyObject(self.obj, self.parent, True) - elif t == 'def': - for name in self._parse_function_doc()[1].split(): - try: - bltn_obj = create(getattr(_builtins, name), builtin, module=builtin) - except AttributeError: - continue + def execute_function(self, evaluator, params): + for name in self._parse_function_doc()[1].split(): + try: + bltn_obj = create(getattr(_builtins, name), builtin, module=builtin) + except AttributeError: + continue + else: + if isinstance(bltn_obj, PyObject): + yield bltn_obj else: - if isinstance(bltn_obj, PyObject): - yield bltn_obj - else: - for result in evaluator.execute(bltn_obj, params): - yield result + for result in evaluator.execute(bltn_obj, params): + yield result def get_self_attributes(self): return [] # Instance compatibility diff --git a/jedi/evaluate/representation.py b/jedi/evaluate/representation.py index bc834834..79916736 100644 --- a/jedi/evaluate/representation.py +++ b/jedi/evaluate/representation.py @@ -87,6 +87,8 @@ class Instance(use_metaclass(CachedMetaClass, Executable)): n.names = n.names[1:] names.append(InstanceElement(self._evaluator, self, n)) + if isinstance(self.base, compiled.PyObject): + return [] names = [] # This loop adds the names of the self object, copies them and removes # the self. From e7c7bbca799dedd46d90f032cfc6aadf65caefec Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Sat, 11 Jan 2014 01:58:31 +0100 Subject: [PATCH 28/52] simple debugging improvement - make it more readable by giving it an indent --- jedi/debug.py | 23 +++++++++++++++++++---- jedi/evaluate/__init__.py | 2 ++ 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/jedi/debug.py b/jedi/debug.py index c6aaf7be..16d7af3d 100644 --- a/jedi/debug.py +++ b/jedi/debug.py @@ -27,8 +27,20 @@ ignored_modules = ['jedi.evaluate.builtin', 'jedi.parser'] def reset_time(): - global start_time + global start_time, debug_indent start_time = time.time() + debug_indent = -1 + + +def increase_indent(func): + """Decorator for makin """ + def wrapper(*args, **kwargs): + global debug_indent + debug_indent += 1 + result = func(*args, **kwargs) + debug_indent -= 1 + return result + return wrapper def dbg(*args): @@ -37,18 +49,21 @@ def dbg(*args): frm = inspect.stack()[1] mod = inspect.getmodule(frm[0]) if not (mod.__name__ in ignored_modules): - debug_function(NOTICE, 'dbg: ' + ', '.join(u(a) for a in args)) + i = ' ' * debug_indent + debug_function(NOTICE, i + 'dbg: ' + ', '.join(u(a) for a in args)) def warning(*args): if debug_function and enable_warning: - debug_function(WARNING, 'warning: ' + ', '.join(u(a) for a in args)) + i = ' ' * debug_indent + debug_function(WARNING, i + 'warning: ' + ', '.join(u(a) for a in args)) def speed(name): if debug_function and enable_speed: now = time.time() - debug_function(SPEED, 'speed: ' + '%s %s' % (name, now - start_time)) + i = ' ' * debug_indent + debug_function(SPEED, i + 'speed: ' + '%s %s' % (name, now - start_time)) def print_to_stdout(level, str_out): diff --git a/jedi/evaluate/__init__.py b/jedi/evaluate/__init__.py index 978202a4..bee22970 100644 --- a/jedi/evaluate/__init__.py +++ b/jedi/evaluate/__init__.py @@ -193,6 +193,7 @@ class Evaluator(object): @memoize_default(default=(), evaluator_is_first_arg=True) @recursion.recursion_decorator + @debug.increase_indent def eval_statement(self, stmt, seek_name=None): """ The starting point of the completion. A statement always owns a call @@ -377,6 +378,7 @@ class Evaluator(object): result = imports.strip_imports(self, types) return self.follow_path(path, set(result), scope, position=position) + @debug.increase_indent def execute(self, obj, params=(), evaluate_generator=False): if obj.isinstance(er.Function): obj = obj.get_decorated_func() From 28ab937ecaba980ab033eb00d39108b6571fad67 Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Sat, 11 Jan 2014 02:55:50 +0100 Subject: [PATCH 29/52] in the process... --- jedi/evaluate/__init__.py | 1 + jedi/evaluate/compiled/__init__.py | 11 ++++++----- jedi/evaluate/compiled/fake.py | 10 +++++++--- jedi/evaluate/representation.py | 7 +++++-- 4 files changed, 19 insertions(+), 10 deletions(-) diff --git a/jedi/evaluate/__init__.py b/jedi/evaluate/__init__.py index bee22970..0c67c28f 100644 --- a/jedi/evaluate/__init__.py +++ b/jedi/evaluate/__init__.py @@ -411,6 +411,7 @@ class Evaluator(object): debug.warning("no __call__ func available", obj) else: debug.warning("no execution possible", obj) + raise NotImplementedError() else: stmts = er.FunctionExecution(self, obj, params).get_return_types(evaluate_generator) diff --git a/jedi/evaluate/compiled/__init__.py b/jedi/evaluate/compiled/__init__.py index 390716e2..3c7e0fcf 100644 --- a/jedi/evaluate/compiled/__init__.py +++ b/jedi/evaluate/compiled/__init__.py @@ -34,7 +34,7 @@ class PyObject(Base): def get_parent_until(self, *args, **kwargs): # compiled modules only use functions and classes/methods (2 levels) - return getattr(self.parent, 'parent', self.parent) or self + return getattr(self.parent, 'parent', self.parent) or self.parent or self @underscore_memoization def _parse_function_doc(self): @@ -74,7 +74,8 @@ class PyObject(Base): def get_subscope_by_name(self, name): if name in dir(self._cls().obj): - return PyName(self._cls, name).parent + print PyName(self._cls(), name).parent + return PyName(self._cls(), name).parent else: raise KeyError("CompiledObject doesn't have an attribute '%s'." % name) @@ -116,14 +117,14 @@ class PyName(object): @underscore_memoization def parent(self): try: - # this has a builtin_function_or_method - return create(getattr(self._obj.obj, self._name), self._obj, - module=self._obj.get_parent_until()) + o = getattr(self._obj.obj, self._name) except AttributeError: # happens e.g. in properties of # PyQt4.QtGui.QStyleOptionComboBox.currentText # -> just set it to None return PyObject(None, builtin) + else: + return create(o, self._obj, module=self._obj.get_parent_until()) @property def names(self): diff --git a/jedi/evaluate/compiled/fake.py b/jedi/evaluate/compiled/fake.py index 61504159..2956d2f9 100644 --- a/jedi/evaluate/compiled/fake.py +++ b/jedi/evaluate/compiled/fake.py @@ -75,7 +75,7 @@ def _load_module(module): with open(os.path.join(path, 'fake', module_name) + '.pym') as f: source = f.read() except IOError: - return {} + return module = Parser(source, module_name).module modules[module_name] = module return module @@ -88,6 +88,8 @@ def get_faked(module, obj): return s mod = _load_module(module) + if mod is None: + return # Having the module as a `parser.representation.module`, we need to scan # for methods. @@ -97,8 +99,10 @@ def get_faked(module, obj): return from_scope(mod, obj) elif not inspect.isclass(obj): # object is a method or descriptor - cls = obj.__objclass__ - return from_scope(from_scope(mod, cls), obj) + cls = from_scope(mod, obj.__objclass__) + if cls is None: + return + return from_scope(cls, obj) def is_class_instance(obj): diff --git a/jedi/evaluate/representation.py b/jedi/evaluate/representation.py index 79916736..1c60a97d 100644 --- a/jedi/evaluate/representation.py +++ b/jedi/evaluate/representation.py @@ -122,6 +122,7 @@ class Instance(use_metaclass(CachedMetaClass, Executable)): def get_subscope_by_name(self, name): sub = self.base.get_subscope_by_name(name) + print sub return InstanceElement(self._evaluator, self, sub, True) def execute_subscope_by_name(self, name, args=()): @@ -199,11 +200,13 @@ class InstanceElement(use_metaclass(CachedMetaClass, pr.Base)): def parent(self): par = self.var.parent if isinstance(par, Class) and par == self.instance.base \ - or isinstance(par, pr.Class) \ + or isinstance(par, pr.Class) \ and par == self.instance.base.base: par = self.instance - elif not isinstance(par, pr.Module): + elif not isinstance(par, (pr.Module, compiled.PyObject)): + print 'HA', par, self.var par = InstanceElement(self.instance._evaluator, self.instance, par, self.is_class_var) + print 'H2A', par return par def get_parent_until(self, *args, **kwargs): From 6f9d834a93e6e9cee265137494a6a8e6a336ab80 Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Sat, 11 Jan 2014 13:01:09 +0100 Subject: [PATCH 30/52] now able to execute instance subscopes on compiled --- jedi/evaluate/__init__.py | 1 - jedi/evaluate/compiled/__init__.py | 55 +++++++++++++++--------------- jedi/evaluate/compiled/fake.py | 49 ++++++++++++++++++-------- jedi/evaluate/representation.py | 3 -- 4 files changed, 62 insertions(+), 46 deletions(-) diff --git a/jedi/evaluate/__init__.py b/jedi/evaluate/__init__.py index 0c67c28f..bee22970 100644 --- a/jedi/evaluate/__init__.py +++ b/jedi/evaluate/__init__.py @@ -411,7 +411,6 @@ class Evaluator(object): debug.warning("no __call__ func available", obj) else: debug.warning("no execution possible", obj) - raise NotImplementedError() else: stmts = er.FunctionExecution(self, obj, params).get_return_types(evaluate_generator) diff --git a/jedi/evaluate/compiled/__init__.py b/jedi/evaluate/compiled/__init__.py index 3c7e0fcf..990cacd9 100644 --- a/jedi/evaluate/compiled/__init__.py +++ b/jedi/evaluate/compiled/__init__.py @@ -23,10 +23,9 @@ class PyObject(Base): start_pos = 0, 0 asserts = [] - def __init__(self, obj, parent=None, instantiated=False): + def __init__(self, obj, parent=None): self.obj = obj self.parent = parent - self.instantiated = instantiated self.doc = inspect.getdoc(obj) def __repr__(self): @@ -74,7 +73,6 @@ class PyObject(Base): def get_subscope_by_name(self, name): if name in dir(self._cls().obj): - print PyName(self._cls(), name).parent return PyName(self._cls(), name).parent else: raise KeyError("CompiledObject doesn't have an attribute '%s'." % name) @@ -87,7 +85,7 @@ class PyObject(Base): def execute_function(self, evaluator, params): for name in self._parse_function_doc()[1].split(): try: - bltn_obj = create(getattr(_builtins, name), builtin, module=builtin) + bltn_obj = _create_from_name(builtin, builtin, name) except AttributeError: continue else: @@ -116,15 +114,8 @@ class PyName(object): @property @underscore_memoization def parent(self): - try: - o = getattr(self._obj.obj, self._name) - except AttributeError: - # happens e.g. in properties of - # PyQt4.QtGui.QStyleOptionComboBox.currentText - # -> just set it to None - return PyObject(None, builtin) - else: - return create(o, self._obj, module=self._obj.get_parent_until()) + module = self._obj.get_parent_until() + return _create_from_name(module, self._obj, self._name) @property def names(self): @@ -236,23 +227,31 @@ builtin = PyObject(_builtins) magic_function_class = PyObject(type(load_module), parent=builtin) -def create(obj, parent=builtin, instantiated=False, module=None): - if not inspect.ismodule(obj): - if module is None: - module = obj.__class__ if fake.is_class_instance(obj) else obj - if not (inspect.isbuiltin(module) or inspect.isclass(module)): - module = obj.__objclass__ - try: - imp_plz = obj.__module__ - except AttributeError: - # Unfortunately in some cases like `int` there's no __module__ - module = builtin - else: - module = PyObject(__import__(imp_plz)) +def _create_from_name(module, parent, name): + faked = fake.get_faked(module.obj, parent, name) + if faked is not None: + faked.parent = parent + return faked - faked = fake.get_faked(module.obj, obj) + try: + obj = getattr(parent, name) + except AttributeError: + # happens e.g. in properties of + # PyQt4.QtGui.QStyleOptionComboBox.currentText + # -> just set it to None + obj = None + return PyObject(obj, parent) + + +def create(obj, parent=builtin, module=None): + """ + A very weird interface class to this module. The more options provided the + more acurate loading compiled objects is. + """ + if not inspect.ismodule(parent): + faked = fake.get_faked(module and module.obj, obj) if faked is not None: faked.parent = parent return faked - return PyObject(obj, parent, instantiated) + return PyObject(obj, parent) diff --git a/jedi/evaluate/compiled/fake.py b/jedi/evaluate/compiled/fake.py index 2956d2f9..7b464487 100644 --- a/jedi/evaluate/compiled/fake.py +++ b/jedi/evaluate/compiled/fake.py @@ -8,7 +8,7 @@ import re import os import inspect -from jedi._compatibility import is_py3k +from jedi._compatibility import is_py3k, builtins from jedi.parser import Parser modules = {} @@ -67,6 +67,7 @@ def _load_module(module): module_name = module.__name__ if module_name == '__builtin__' and not is_py3k: module_name = 'builtins' + try: return modules[module_name] except KeyError: @@ -81,28 +82,48 @@ def _load_module(module): return module -def get_faked(module, obj): - def from_scope(scope, obj): +def get_faked(module, obj, name=None): + def from_scope(scope, obj_name): for s in scope.subscopes: - if str(s.name) == obj.__name__: + if str(s.name) == obj_name: return s + # Crazy underscore actions to try to escape all the internal madness. + obj = obj.__class__ if is_class_instance(obj) else obj + if module is None: + try: + module = obj.__objclass__ + except AttributeError: + pass + + try: + imp_plz = obj.__module__ + except AttributeError: + # Unfortunately in some cases like `int` there's no __module__ + module = builtins + else: + module = __import__(imp_plz) + mod = _load_module(module) if mod is None: return # Having the module as a `parser.representation.module`, we need to scan # for methods. - if is_class_instance(obj): - obj = obj.__class__ - if inspect.isbuiltin(obj): - return from_scope(mod, obj) - elif not inspect.isclass(obj): - # object is a method or descriptor - cls = from_scope(mod, obj.__objclass__) - if cls is None: - return - return from_scope(cls, obj) + if name is None: + if inspect.isbuiltin(obj): + return from_scope(mod, obj.__name__) + elif not inspect.isclass(obj): + # object is a method or descriptor + cls = from_scope(mod, obj.__objclass__.__name__) + if cls is None: + return + return from_scope(cls, obj.__name__) + else: + if obj == module: + return from_scope(mod, name) + else: + return from_scope(mod, name) def is_class_instance(obj): diff --git a/jedi/evaluate/representation.py b/jedi/evaluate/representation.py index 1c60a97d..960c211d 100644 --- a/jedi/evaluate/representation.py +++ b/jedi/evaluate/representation.py @@ -122,7 +122,6 @@ class Instance(use_metaclass(CachedMetaClass, Executable)): def get_subscope_by_name(self, name): sub = self.base.get_subscope_by_name(name) - print sub return InstanceElement(self._evaluator, self, sub, True) def execute_subscope_by_name(self, name, args=()): @@ -204,9 +203,7 @@ class InstanceElement(use_metaclass(CachedMetaClass, pr.Base)): and par == self.instance.base.base: par = self.instance elif not isinstance(par, (pr.Module, compiled.PyObject)): - print 'HA', par, self.var par = InstanceElement(self.instance._evaluator, self.instance, par, self.is_class_var) - print 'H2A', par return par def get_parent_until(self, *args, **kwargs): From fc35e69a161c1d4acda74b361fa5485e24aca88b Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Sat, 11 Jan 2014 13:41:03 +0100 Subject: [PATCH 31/52] adding helpers.FakeName and other fakes to make it easier to fake parser names and statements --- jedi/evaluate/compiled/__init__.py | 2 +- jedi/evaluate/helpers.py | 16 ++++++++++++++++ jedi/evaluate/iterable.py | 10 ++-------- jedi/evaluate/param.py | 19 +++++-------------- 4 files changed, 24 insertions(+), 23 deletions(-) diff --git a/jedi/evaluate/compiled/__init__.py b/jedi/evaluate/compiled/__init__.py index 990cacd9..34a45b75 100644 --- a/jedi/evaluate/compiled/__init__.py +++ b/jedi/evaluate/compiled/__init__.py @@ -59,7 +59,7 @@ class PyObject(Base): def _cls(self): # Ensures that a PyObject is returned that is not an instance (like list) if fake.is_class_instance(self.obj): - return PyObject(self.obj.__class__, self.parent, True) + return PyObject(self.obj.__class__, self.parent) return self def get_defined_names(self): diff --git a/jedi/evaluate/helpers.py b/jedi/evaluate/helpers.py index cf2cf916..77a9e70a 100644 --- a/jedi/evaluate/helpers.py +++ b/jedi/evaluate/helpers.py @@ -156,3 +156,19 @@ def scan_statement_for_calls(stmt, search_name, assignment_details=False): s_new = s_new.next return result + + +class FakeSubModule(): + line_offset = 0 + + +class FakeStatement(pr.Statement): + def __init__(self, token_list): + p = 0, 0 + super(FakeStatement, self).__init__(FakeSubModule, token_list, p, p) + + +class FakeName(pr.Name): + def __init__(self, name, parent=None): + p = 0, 0 + super(FakeName, self).__init__(FakeSubModule, [(name, p)], p, p, parent) diff --git a/jedi/evaluate/iterable.py b/jedi/evaluate/iterable.py index 985b08b5..6a78c914 100644 --- a/jedi/evaluate/iterable.py +++ b/jedi/evaluate/iterable.py @@ -24,16 +24,10 @@ class Generator(use_metaclass(CachedMetaClass, pr.Base)): content of a generator. """ names = [] - none_pos = (0, 0) executes_generator = ('__next__', 'send') for n in ('close', 'throw') + executes_generator: - name = pr.Name(compiled.builtin, [(n, none_pos)], - none_pos, none_pos) - if n in executes_generator: - name.parent = self - else: - name.parent = compiled.builtin - names.append(name) + parent = self if n in executes_generator else compiled.builtin + names.append(helpers.FakeName(n, parent)) debug.dbg('generator names', names) return names diff --git a/jedi/evaluate/param.py b/jedi/evaluate/param.py index c363ed61..829401da 100644 --- a/jedi/evaluate/param.py +++ b/jedi/evaluate/param.py @@ -3,6 +3,7 @@ import copy from jedi.parser import representation as pr from jedi.evaluate import iterable from jedi.evaluate import common +from jedi.evaluate import helpers def get_params(evaluator, func, var_args): @@ -23,11 +24,11 @@ def get_params(evaluator, func, var_args): new_param.parent = parent # create an Array (-> needed for *args/**kwargs tuples/dicts) - arr = pr.Array(_FakeSubModule, start_pos, array_type, parent) + arr = pr.Array(helpers.FakeSubModule, start_pos, array_type, parent) arr.values = values key_stmts = [] for key in keys: - stmt = pr.Statement(_FakeSubModule, [], start_pos, None) + stmt = pr.Statement(helpers.FakeSubModule, [], start_pos, None) stmt._expression_list = [key] key_stmts.append(stmt) arr.keys = key_stmts @@ -139,7 +140,7 @@ def _var_args_iterator(evaluator, var_args): continue old = stmt # generate a statement if it's not already one. - stmt = pr.Statement(_FakeSubModule, [], (0, 0), None) + stmt = pr.Statement(helpers.FakeSubModule, [], (0, 0), None) stmt._expression_list = [old] # *args @@ -154,7 +155,7 @@ def _var_args_iterator(evaluator, var_args): yield None, field_stmt elif isinstance(array, iterable.Generator): for field_stmt in array.iter_content(): - yield None, _FakeStatement(field_stmt) + yield None, helpers._FakeStatement([field_stmt]) # **kwargs elif expression_list[0] == '**': for array in evaluator.eval_expression_list(expression_list[1:]): @@ -175,13 +176,3 @@ def _var_args_iterator(evaluator, var_args): yield key_arr[0].name, stmt else: yield None, stmt - - -class _FakeSubModule(): - line_offset = 0 - - -class _FakeStatement(pr.Statement): - def __init__(self, content): - p = 0, 0 - super(_FakeStatement, self).__init__(_FakeSubModule, [content], p, p) From bfe0c62e7f1b326d0c875597cbd1bc92568e9948 Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Sat, 11 Jan 2014 15:58:14 +0100 Subject: [PATCH 32/52] filter None (is more a keyword than a builtin object) --- jedi/evaluate/compiled/__init__.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/jedi/evaluate/compiled/__init__.py b/jedi/evaluate/compiled/__init__.py index 34a45b75..ac404ce8 100644 --- a/jedi/evaluate/compiled/__init__.py +++ b/jedi/evaluate/compiled/__init__.py @@ -105,24 +105,24 @@ class PyObject(Base): class PyName(object): def __init__(self, obj, name): self._obj = obj - self._name = name + self.name = name self.start_pos = 0, 0 # an illegal start_pos, to make sorting easy. def __repr__(self): - return '<%s: (%s).%s>' % (type(self).__name__, repr(self._obj.obj), self._name) + return '<%s: (%s).%s>' % (type(self).__name__, repr(self._obj.obj), self.name) @property @underscore_memoization def parent(self): module = self._obj.get_parent_until() - return _create_from_name(module, self._obj, self._name) + return _create_from_name(module, self._obj, self.name) @property def names(self): - return [self._name] # compatibility with parser.representation.Name + return [self.name] # compatibility with parser.representation.Name def get_code(self): - return self._name + return self.name def load_module(path, name): @@ -223,7 +223,14 @@ def _parse_function_doc(doc): return param_str, ret -builtin = PyObject(_builtins) +class Builtin(PyObject): + def get_defined_names(self): + # Filter None, because it's really just a keyword, nobody wants to + # access it. + return [d for d in super(Builtin, self).get_defined_names() if d.name != 'None'] + + +builtin = Builtin(_builtins) magic_function_class = PyObject(type(load_module), parent=builtin) From c6a14a348e4294642252e259f8162caee1aa8945 Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Sat, 11 Jan 2014 16:04:05 +0100 Subject: [PATCH 33/52] parent was wrongly used in _create_from_name --- jedi/evaluate/compiled/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jedi/evaluate/compiled/__init__.py b/jedi/evaluate/compiled/__init__.py index ac404ce8..ee26a5c7 100644 --- a/jedi/evaluate/compiled/__init__.py +++ b/jedi/evaluate/compiled/__init__.py @@ -235,13 +235,13 @@ magic_function_class = PyObject(type(load_module), parent=builtin) def _create_from_name(module, parent, name): - faked = fake.get_faked(module.obj, parent, name) + faked = fake.get_faked(module.obj, parent.obj, name) if faked is not None: faked.parent = parent return faked try: - obj = getattr(parent, name) + obj = getattr(parent.obj, name) except AttributeError: # happens e.g. in properties of # PyQt4.QtGui.QStyleOptionComboBox.currentText From d430ef53a78f76b4d611346c3dc1a0c0ac42c53e Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Sat, 11 Jan 2014 16:14:58 +0100 Subject: [PATCH 34/52] fix a few more minor issues --- jedi/evaluate/imports.py | 2 +- jedi/evaluate/iterable.py | 2 +- jedi/evaluate/param.py | 2 +- jedi/parser/representation.py | 6 +----- 4 files changed, 4 insertions(+), 8 deletions(-) diff --git a/jedi/evaluate/imports.py b/jedi/evaluate/imports.py index d1020615..61fec297 100644 --- a/jedi/evaluate/imports.py +++ b/jedi/evaluate/imports.py @@ -436,7 +436,7 @@ def get_modules_containing_name(mods, name): return load_module(path, source) # skip non python modules - mods = set(m for m in mods if m.path is None or m.path.endswith('.py')) + mods = set(m for m in mods if not isinstance(m, compiled.PyObject)) mod_paths = set() for m in mods: mod_paths.add(m.path) diff --git a/jedi/evaluate/iterable.py b/jedi/evaluate/iterable.py index 6a78c914..842cac82 100644 --- a/jedi/evaluate/iterable.py +++ b/jedi/evaluate/iterable.py @@ -228,7 +228,7 @@ def _check_array_additions(evaluator, compare_array, module, is_list): >>> a = [""] >>> a.append(1) """ - if not settings.dynamic_array_additions or module.is_builtin(): + if not settings.dynamic_array_additions or isinstance(module, compiled.PyObject): return [] def check_calls(calls, add_name): diff --git a/jedi/evaluate/param.py b/jedi/evaluate/param.py index 829401da..e6c2ee45 100644 --- a/jedi/evaluate/param.py +++ b/jedi/evaluate/param.py @@ -155,7 +155,7 @@ def _var_args_iterator(evaluator, var_args): yield None, field_stmt elif isinstance(array, iterable.Generator): for field_stmt in array.iter_content(): - yield None, helpers._FakeStatement([field_stmt]) + yield None, helpers.FakeStatement([field_stmt]) # **kwargs elif expression_list[0] == '**': for array in evaluator.eval_expression_list(expression_list[1:]): diff --git a/jedi/parser/representation.py b/jedi/parser/representation.py index 10e9882b..60973ade 100644 --- a/jedi/parser/representation.py +++ b/jedi/parser/representation.py @@ -362,17 +362,13 @@ class SubModule(Scope, Module): string = '' # no path -> empty name else: sep = (re.escape(os.path.sep),) * 2 - r = re.search(r'([^%s]*?)(%s__init__)?(\.py|\.so)?$' % sep, - self.path) + r = re.search(r'([^%s]*?)(%s__init__)?(\.py|\.so)?$' % sep, self.path) # remove PEP 3149 names string = re.sub('\.[a-z]+-\d{2}[mud]{0,3}$', '', r.group(1)) # positions are not real therefore choose (0, 0) names = [(string, (0, 0))] return Name(self, names, (0, 0), (0, 0), self.use_as_parent) - def is_builtin(self): - return not (self.path is None or self.path.endswith('.py')) - @property def has_explicit_absolute_import(self): """ From 1765fadf730fec34ca677b914fcb83d1767fbbc9 Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Sat, 11 Jan 2014 18:05:44 +0100 Subject: [PATCH 35/52] fix problems with self attributes - from fake modules --- jedi/debug.py | 1 + jedi/evaluate/__init__.py | 15 +++++++------- jedi/evaluate/compiled/__init__.py | 18 ++++++++++++++++ jedi/evaluate/compiled/fake.py | 26 ++++++++++++++++-------- jedi/evaluate/compiled/fake/_weakref.pym | 2 +- jedi/evaluate/representation.py | 9 ++++---- test/test_compiled.py | 5 ++++- 7 files changed, 53 insertions(+), 23 deletions(-) diff --git a/jedi/debug.py b/jedi/debug.py index 16d7af3d..b297124b 100644 --- a/jedi/debug.py +++ b/jedi/debug.py @@ -24,6 +24,7 @@ enable_notice = False # callback, interface: level, str debug_function = None ignored_modules = ['jedi.evaluate.builtin', 'jedi.parser'] +debug_indent = -1 def reset_time(): diff --git a/jedi/evaluate/__init__.py b/jedi/evaluate/__init__.py index bee22970..8ab470db 100644 --- a/jedi/evaluate/__init__.py +++ b/jedi/evaluate/__init__.py @@ -389,7 +389,7 @@ class Evaluator(object): except stdlib.NotInStdLib: pass - if isinstance(obj, compiled.PyObject): + if obj.isinstance(compiled.PyObject): if obj.is_executable_class(): return [er.Instance(self, obj, params)] else: @@ -401,9 +401,9 @@ class Evaluator(object): return obj.iter_content() else: stmts = [] - try: - obj.returns # Test if it is a function - except AttributeError: + if obj.isinstance(er.Function): + stmts = er.FunctionExecution(self, obj, params).get_return_types(evaluate_generator) + else: if hasattr(obj, 'execute_subscope_by_name'): try: stmts = obj.execute_subscope_by_name('__call__', params) @@ -411,8 +411,6 @@ class Evaluator(object): debug.warning("no __call__ func available", obj) else: debug.warning("no execution possible", obj) - else: - stmts = er.FunctionExecution(self, obj, params).get_return_types(evaluate_generator) debug.dbg('execute result: %s in %s' % (stmts, obj)) return imports.strip_imports(self, stmts) @@ -454,8 +452,9 @@ def filter_private_variable(scope, call_scope, var_name): if isinstance(var_name, (str, unicode)) and isinstance(scope, er.Instance)\ and var_name.startswith('__') and not var_name.endswith('__'): s = call_scope.get_parent_until((pr.Class, er.Instance)) - if s != scope and s != scope.base.base: - return True + if not isinstance(scope.base, compiled.PyObject): + if s != scope and s != scope.base.base: + return True return False diff --git a/jedi/evaluate/compiled/__init__.py b/jedi/evaluate/compiled/__init__.py index ee26a5c7..fbb9e1de 100644 --- a/jedi/evaluate/compiled/__init__.py +++ b/jedi/evaluate/compiled/__init__.py @@ -83,6 +83,8 @@ class PyObject(Base): return self._cls().obj.__name__ def execute_function(self, evaluator, params): + if self.type() != 'def': + return for name in self._parse_function_doc()[1].split(): try: bltn_obj = _create_from_name(builtin, builtin, name) @@ -95,6 +97,21 @@ class PyObject(Base): for result in evaluator.execute(bltn_obj, params): yield result + @property + @underscore_memoization + def subscopes(self): + """ + Returns only the faked scopes - the other ones are not important for + internal analysis. + """ + module = self.get_parent_until() + faked_subscopes = [] + for name in dir(self._cls().obj): + f = fake.get_faked(module.obj, self.obj, name) + if f: + faked_subscopes.append(f) + return faked_subscopes + def get_self_attributes(self): return [] # Instance compatibility @@ -236,6 +253,7 @@ magic_function_class = PyObject(type(load_module), parent=builtin) def _create_from_name(module, parent, name): faked = fake.get_faked(module.obj, parent.obj, name) + # only functions are necessary. if faked is not None: faked.parent = parent return faked diff --git a/jedi/evaluate/compiled/fake.py b/jedi/evaluate/compiled/fake.py index 7b464487..65367b10 100644 --- a/jedi/evaluate/compiled/fake.py +++ b/jedi/evaluate/compiled/fake.py @@ -10,6 +10,7 @@ import inspect from jedi._compatibility import is_py3k, builtins from jedi.parser import Parser +from jedi.parser.representation import Class modules = {} @@ -63,7 +64,7 @@ def _load_fakes(module_name): return mixin_dct -def _load_module(module): +def _load_faked_module(module): module_name = module.__name__ if module_name == '__builtin__' and not is_py3k: module_name = 'builtins' @@ -82,7 +83,7 @@ def _load_module(module): return module -def get_faked(module, obj, name=None): +def _faked(module, obj, name=None): def from_scope(scope, obj_name): for s in scope.subscopes: if str(s.name) == obj_name: @@ -104,26 +105,35 @@ def get_faked(module, obj, name=None): else: module = __import__(imp_plz) - mod = _load_module(module) - if mod is None: + faked_mod = _load_faked_module(module) + if faked_mod is None: return # Having the module as a `parser.representation.module`, we need to scan # for methods. if name is None: if inspect.isbuiltin(obj): - return from_scope(mod, obj.__name__) + return from_scope(faked_mod, obj.__name__) elif not inspect.isclass(obj): # object is a method or descriptor - cls = from_scope(mod, obj.__objclass__.__name__) + cls = from_scope(faked_mod, obj.__objclass__.__name__) if cls is None: return return from_scope(cls, obj.__name__) else: if obj == module: - return from_scope(mod, name) + return from_scope(faked_mod, name) else: - return from_scope(mod, name) + cls = from_scope(faked_mod, obj.__name__) + if cls is None: + return + return from_scope(cls, name) + + +def get_faked(*args, **kwargs): + result = _faked(*args, **kwargs) + if not isinstance(result, Class): + return result def is_class_instance(obj): diff --git a/jedi/evaluate/compiled/fake/_weakref.pym b/jedi/evaluate/compiled/fake/_weakref.pym index 05eab2c8..8d21a2c4 100644 --- a/jedi/evaluate/compiled/fake/_weakref.pym +++ b/jedi/evaluate/compiled/fake/_weakref.pym @@ -1,7 +1,7 @@ def proxy(object, callback=None): return object -class ref(): +class weakref(): def __init__(self, object, callback=None): self.__object = object def __call__(self): diff --git a/jedi/evaluate/representation.py b/jedi/evaluate/representation.py index 960c211d..df9ff143 100644 --- a/jedi/evaluate/representation.py +++ b/jedi/evaluate/representation.py @@ -87,8 +87,6 @@ class Instance(use_metaclass(CachedMetaClass, Executable)): n.names = n.names[1:] names.append(InstanceElement(self._evaluator, self, n)) - if isinstance(self.base, compiled.PyObject): - return [] names = [] # This loop adds the names of the self object, copies them and removes # the self. @@ -115,9 +113,10 @@ class Instance(use_metaclass(CachedMetaClass, Executable)): if n.names[0] == self_name and len(n.names) == 2: add_self_dot_name(n) - for s in self.base.get_super_classes(): - for inst in self._evaluator.execute(s): - names += inst.get_self_attributes() + if not isinstance(self.base, compiled.PyObject): + for s in self.base.get_super_classes(): + for inst in self._evaluator.execute(s): + names += inst.get_self_attributes() return names def get_subscope_by_name(self, name): diff --git a/test/test_compiled.py b/test/test_compiled.py index 287549ee..88eb8ba9 100644 --- a/test/test_compiled.py +++ b/test/test_compiled.py @@ -13,8 +13,11 @@ def test_simple(): objs = list(e.execute(upper[0])) assert len(objs) == 1 assert objs[0].obj is str - assert objs[0].instantiated is True def test_fake_loading(): assert isinstance(compiled.create(next), Function) + + string = compiled.builtin.get_subscope_by_name('str') + from_name = compiled._create_from_name(compiled.builtin, string, '__init__') + assert isinstance(from_name, Function) From 4006b231d353b70274e39490d13e6acdf576e723 Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Sun, 12 Jan 2014 00:33:13 +0100 Subject: [PATCH 36/52] fix a few last standing issues with integration tests. ok after running tests with tox i see that they are not the last issues... --- jedi/api/__init__.py | 2 +- jedi/evaluate/__init__.py | 8 ++++---- jedi/evaluate/compiled/__init__.py | 1 + jedi/evaluate/helpers.py | 5 +++-- jedi/evaluate/iterable.py | 2 ++ 5 files changed, 11 insertions(+), 7 deletions(-) diff --git a/jedi/api/__init__.py b/jedi/api/__init__.py index d8451662..25b0d836 100644 --- a/jedi/api/__init__.py +++ b/jedi/api/__init__.py @@ -143,7 +143,7 @@ class Script(object): if not dot: # add named params for call_def in self.call_signatures(): - if not call_def.module.is_builtin(): + if not isinstance(call_def.module, compiled.PyObject): for p in call_def.params: completions.append((p.get_name(), p)) diff --git a/jedi/evaluate/__init__.py b/jedi/evaluate/__init__.py index 8ab470db..36a410e7 100644 --- a/jedi/evaluate/__init__.py +++ b/jedi/evaluate/__init__.py @@ -282,8 +282,8 @@ class Evaluator(object): result += self.eval_call(call) elif call == '*': if [r for r in result if isinstance(r, iterable.Array) - or isinstance(r, er.Instance) - and str(r.name) == 'str']: + or isinstance(r, compiled.PyObject) + and isinstance(r.obj, (str, unicode))]: # if it is an iterable, ignore * operations next(calls_iterator) return set(result) @@ -452,8 +452,8 @@ def filter_private_variable(scope, call_scope, var_name): if isinstance(var_name, (str, unicode)) and isinstance(scope, er.Instance)\ and var_name.startswith('__') and not var_name.endswith('__'): s = call_scope.get_parent_until((pr.Class, er.Instance)) - if not isinstance(scope.base, compiled.PyObject): - if s != scope and s != scope.base.base: + if s != scope and (isinstance(scope.base, compiled.PyObject) + or s != scope.base.base): return True return False diff --git a/jedi/evaluate/compiled/__init__.py b/jedi/evaluate/compiled/__init__.py index fbb9e1de..78dafecc 100644 --- a/jedi/evaluate/compiled/__init__.py +++ b/jedi/evaluate/compiled/__init__.py @@ -109,6 +109,7 @@ class PyObject(Base): for name in dir(self._cls().obj): f = fake.get_faked(module.obj, self.obj, name) if f: + f.parent = self faked_subscopes.append(f) return faked_subscopes diff --git a/jedi/evaluate/helpers.py b/jedi/evaluate/helpers.py index 77a9e70a..b298d8bb 100644 --- a/jedi/evaluate/helpers.py +++ b/jedi/evaluate/helpers.py @@ -163,9 +163,10 @@ class FakeSubModule(): class FakeStatement(pr.Statement): - def __init__(self, token_list): + def __init__(self, expression_list): p = 0, 0 - super(FakeStatement, self).__init__(FakeSubModule, token_list, p, p) + super(FakeStatement, self).__init__(FakeSubModule, expression_list, p, p) + self._expression_list = expression_list class FakeName(pr.Name): diff --git a/jedi/evaluate/iterable.py b/jedi/evaluate/iterable.py index 842cac82..63aeb801 100644 --- a/jedi/evaluate/iterable.py +++ b/jedi/evaluate/iterable.py @@ -96,6 +96,8 @@ class Array(use_metaclass(CachedMetaClass, pr.Base)): str_key = key.value elif isinstance(key, pr.Name): str_key = str(key) + else: + continue if mixed_index == str_key: index = i From b93c761db64f5b0e4f3b51464cc69cd6c80ec508 Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Sun, 12 Jan 2014 01:11:59 +0100 Subject: [PATCH 37/52] after fixing private variable filtering it looks much better --- jedi/evaluate/__init__.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/jedi/evaluate/__init__.py b/jedi/evaluate/__init__.py index 36a410e7..247046ea 100644 --- a/jedi/evaluate/__init__.py +++ b/jedi/evaluate/__init__.py @@ -451,10 +451,16 @@ def filter_private_variable(scope, call_scope, var_name): """private variables begin with a double underline `__`""" if isinstance(var_name, (str, unicode)) and isinstance(scope, er.Instance)\ and var_name.startswith('__') and not var_name.endswith('__'): - s = call_scope.get_parent_until((pr.Class, er.Instance)) - if s != scope and (isinstance(scope.base, compiled.PyObject) - or s != scope.base.base): - return True + s = call_scope.get_parent_until((pr.Class, er.Instance, compiled.PyObject)) + #if s != scope and (isinstance(scope.base, compiled.PyObject) + # or s != scope.base.base): + if s != scope: + if isinstance(scope.base, compiled.PyObject): + if s != scope.base: + return True + else: + if s != scope.base.base: + return True return False From 0bff729294b92b61f7b307d0c212b56d7c99b53e Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Sun, 12 Jan 2014 02:15:59 +0100 Subject: [PATCH 38/52] lots of small bugfixes --- jedi/api/classes.py | 11 +++++++---- jedi/evaluate/__init__.py | 2 -- jedi/evaluate/compiled/__init__.py | 3 +++ test/test_api.py | 7 +++---- test/test_api_classes.py | 1 + 5 files changed, 14 insertions(+), 10 deletions(-) diff --git a/jedi/api/classes.py b/jedi/api/classes.py index bca6975a..875baba7 100644 --- a/jedi/api/classes.py +++ b/jedi/api/classes.py @@ -138,8 +138,10 @@ class BaseDefinition(object): """ # generate the type stripped = self._definition - if isinstance(self._definition, er.InstanceElement): - stripped = self._definition.var + if isinstance(stripped, compiled.PyObject): + return stripped.type() + if isinstance(stripped, er.InstanceElement): + stripped = stripped.var if isinstance(stripped, pr.Name): stripped = stripped.parent return type(stripped).__name__.lower() @@ -438,7 +440,9 @@ class Definition(BaseDefinition): if isinstance(d, er.InstanceElement): d = d.var - if isinstance(d, pr.Name): + if isinstance(d, compiled.PyObject): + return d.name + elif isinstance(d, pr.Name): return d.names[-1] if d.names else None elif isinstance(d, iterable.Array): return unicode(d.type) @@ -457,7 +461,6 @@ class Definition(BaseDefinition): return d.assignment_details[0][1].values[0][0].name.names[-1] except IndexError: return None - return None @property def description(self): diff --git a/jedi/evaluate/__init__.py b/jedi/evaluate/__init__.py index 247046ea..cff9e5d3 100644 --- a/jedi/evaluate/__init__.py +++ b/jedi/evaluate/__init__.py @@ -452,8 +452,6 @@ def filter_private_variable(scope, call_scope, var_name): if isinstance(var_name, (str, unicode)) and isinstance(scope, er.Instance)\ and var_name.startswith('__') and not var_name.endswith('__'): s = call_scope.get_parent_until((pr.Class, er.Instance, compiled.PyObject)) - #if s != scope and (isinstance(scope.base, compiled.PyObject) - # or s != scope.base.base): if s != scope: if isinstance(scope.base, compiled.PyObject): if s != scope.base: diff --git a/jedi/evaluate/compiled/__init__.py b/jedi/evaluate/compiled/__init__.py index 78dafecc..c8be7770 100644 --- a/jedi/evaluate/compiled/__init__.py +++ b/jedi/evaluate/compiled/__init__.py @@ -126,6 +126,9 @@ class PyName(object): self.name = name self.start_pos = 0, 0 # an illegal start_pos, to make sorting easy. + def get_parent_until(self): + return self.parent.get_parent_until() + def __repr__(self): return '<%s: (%s).%s>' % (type(self).__name__, repr(self._obj.obj), self.name) diff --git a/test/test_api.py b/test/test_api.py index 66e2c5ff..74c81a9e 100644 --- a/test/test_api.py +++ b/test/test_api.py @@ -8,8 +8,8 @@ from pytest import raises def test_preload_modules(): def check_loaded(*modules): - # + 1 for builtin, +1 for None module (currently used) - assert len(new) == len(modules) + 2 + # +1 for None module (currently used) + assert len(new) == len(modules) + 1 for i in modules + ('__builtin__',): assert [i in k for k in new.keys() if k is not None] @@ -20,9 +20,8 @@ def test_preload_modules(): new['__builtin__'] = temp_cache['__builtin__'] api.preload_module('datetime') - check_loaded('datetime') api.preload_module('json', 'token') - check_loaded('datetime', 'json', 'token') + check_loaded('json', 'token') cache.parser_cache = temp_cache diff --git a/test/test_api_classes.py b/test/test_api_classes.py index 6933a1a8..43a99db7 100644 --- a/test/test_api_classes.py +++ b/test/test_api_classes.py @@ -15,6 +15,7 @@ def test_is_keyword(): results = Script('str', 1, 1, None).goto_definitions() assert len(results) == 1 and results[0].is_keyword is False + def make_definitions(): """ Return a list of definitions for parametrized tests. From 7a0dc41b6261dee5c4bdcf8bb8c7ae9a0347f3a5 Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Sun, 12 Jan 2014 02:23:35 +0100 Subject: [PATCH 39/52] fixed doctest issues --- jedi/evaluate/__init__.py | 2 +- jedi/evaluate/compiled/__init__.py | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/jedi/evaluate/__init__.py b/jedi/evaluate/__init__.py index cff9e5d3..e99ce0fc 100644 --- a/jedi/evaluate/__init__.py +++ b/jedi/evaluate/__init__.py @@ -128,7 +128,7 @@ class Evaluator(object): true (default). >>> pairs[2] #doctest: +ELLIPSIS - (, [, ...]) + (, [, ...]) :rtype: [(pr.Scope, [pr.Name])] :return: Return an generator that yields a pair of scope and names. diff --git a/jedi/evaluate/compiled/__init__.py b/jedi/evaluate/compiled/__init__.py index c8be7770..e13fab91 100644 --- a/jedi/evaluate/compiled/__init__.py +++ b/jedi/evaluate/compiled/__init__.py @@ -129,8 +129,11 @@ class PyName(object): def get_parent_until(self): return self.parent.get_parent_until() + def __str__(self): + return self.name + def __repr__(self): - return '<%s: (%s).%s>' % (type(self).__name__, repr(self._obj.obj), self.name) + return '<%s: (%s).%s>' % (type(self).__name__, self._obj.name, self.name) @property @underscore_memoization From bd239446f547107182e74172d558a1fa48e159fe Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Sun, 12 Jan 2014 02:42:00 +0100 Subject: [PATCH 40/52] fix call signatures --- jedi/api/__init__.py | 3 ++- test/test_call_signatures.py | 6 +++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/jedi/api/__init__.py b/jedi/api/__init__.py index 25b0d836..1d0eb873 100644 --- a/jedi/api/__init__.py +++ b/jedi/api/__init__.py @@ -518,7 +518,8 @@ class Script(object): debug.speed('func_call followed') return [classes.CallDef(o, index, call) for o in origins - if o.isinstance(er.Function, er.Instance, er.Class)] + if o.isinstance(er.Function, er.Instance, er.Class) + or isinstance(o, compiled.PyObject) and o.type() != 'module'] def _func_call_and_param_index(self): debug.speed('func_call start') diff --git a/test/test_call_signatures.py b/test/test_call_signatures.py index 9505b3ae..6fe99a9e 100644 --- a/test/test_call_signatures.py +++ b/test/test_call_signatures.py @@ -15,7 +15,7 @@ class TestCallSignatures(TestCase): assert signatures[0].call_name == expected_name assert signatures[0].index == expected_index - def test_call_signatures(self): + def test_simple(self): def run(source, name, index=0, column=None, line=1): self._run(source, name, index, line, column) @@ -72,7 +72,7 @@ class TestCallSignatures(TestCase): "func(alpha='101'," run(s, 'func', 0, column=13, line=2) - def test_function_definition_complex(self): + def test_complex(self): s = """ def abc(a,b): pass @@ -106,7 +106,7 @@ class TestCallSignatures(TestCase): # just don't throw an exception (if numpy doesn't exist, just ignore it) assert Script(s).call_signatures() == [] - def test_function_definition_empty_paren_pre_space(self): + def test_call_signatures_empty_parentheses_pre_space(self): s = textwrap.dedent("""\ def f(a, b): pass From e4f3f5bea27c15f82583bd3dd7bb6bb8e4e4204f Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Sun, 12 Jan 2014 14:28:42 +0100 Subject: [PATCH 41/52] fix recursion issue with compiled classes --- jedi/evaluate/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/jedi/evaluate/__init__.py b/jedi/evaluate/__init__.py index e99ce0fc..a1ce372a 100644 --- a/jedi/evaluate/__init__.py +++ b/jedi/evaluate/__init__.py @@ -146,7 +146,9 @@ class Evaluator(object): if not (scope != non_flow and scope.isinstance(pr.Class) or scope.isinstance(pr.Flow) or scope.isinstance(er.Instance) - and non_flow.isinstance(er.Function)): + and non_flow.isinstance(er.Function) + or isinstance(scope, compiled.PyObject) + and scope.type() == 'class' and in_func_scope != scope): try: if isinstance(scope, er.Instance): for g in scope.scope_generator(): From c75cef088281c1f813b3cab15cac12f709b6c5aa Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Sun, 12 Jan 2014 17:02:26 +0100 Subject: [PATCH 42/52] fix some python 3 compatibility things (which involves some real bugs, but py2 was passing) --- jedi/api/__init__.py | 5 ++--- jedi/api/classes.py | 6 +++--- jedi/evaluate/compiled/__init__.py | 3 +++ jedi/evaluate/compiled/fake.py | 3 --- test/test_api.py | 16 ++++++++-------- 5 files changed, 16 insertions(+), 17 deletions(-) diff --git a/jedi/api/__init__.py b/jedi/api/__init__.py index 1d0eb873..a2f4b89e 100644 --- a/jedi/api/__init__.py +++ b/jedi/api/__init__.py @@ -563,11 +563,10 @@ class Script(object): match = re.match(r'^(.*?)(\.|)(\w?[\w\d]*)$', path, flags=re.S) return match.groups() - @staticmethod - def _sorted_defs(d): + def _sorted_defs(self, d): # Note: `or ''` below is required because `module_path` could be # None and you can't compare None and str in Python 3. - return sorted(d, key=lambda x: (x.module_path or '', x.line, x.column)) + return sorted(d, key=lambda x: (x.module_path or '', x.line or 0, x.column or 0)) class Interpreter(Script): diff --git a/jedi/api/classes.py b/jedi/api/classes.py index 875baba7..c3f5d0b1 100644 --- a/jedi/api/classes.py +++ b/jedi/api/classes.py @@ -173,11 +173,11 @@ class BaseDefinition(object): The module name. >>> from jedi import Script - >>> source = 'import datetime' - >>> script = Script(source, 1, len(source), 'example.py') + >>> source = 'import json' + >>> script = Script(source, path='example.py') >>> d = script.goto_definitions()[0] >>> print(d.module_name) # doctest: +ELLIPSIS - datetime + json """ return str(self._module.name) diff --git a/jedi/evaluate/compiled/__init__.py b/jedi/evaluate/compiled/__init__.py index e13fab91..cddf675f 100644 --- a/jedi/evaluate/compiled/__init__.py +++ b/jedi/evaluate/compiled/__init__.py @@ -154,6 +154,9 @@ def load_module(path, name): name = os.path.basename(path) name = name.rpartition('.')[0] # cut file type (normally .so) + # sometimes there are endings like `_sqlite3.cpython-32mu` + name = re.sub(r'\..*', '', name) + sys_path = get_sys_path() if path: sys_path.insert(0, path) diff --git a/jedi/evaluate/compiled/fake.py b/jedi/evaluate/compiled/fake.py index 65367b10..73734e5c 100644 --- a/jedi/evaluate/compiled/fake.py +++ b/jedi/evaluate/compiled/fake.py @@ -45,9 +45,6 @@ def _load_fakes(module_name): raise NotImplementedError() return funcs - # sometimes there are stupid endings like `_sqlite3.cpython-32mu` - module_name = re.sub(r'\..*', '', module_name) - if module_name == '__builtin__' and not is_py3k: module_name = 'builtins' path = os.path.dirname(os.path.abspath(__file__)) diff --git a/test/test_api.py b/test/test_api.py index 74c81a9e..44f94c7b 100644 --- a/test/test_api.py +++ b/test/test_api.py @@ -2,24 +2,23 @@ Test all things related to the ``jedi.api`` module. """ -from jedi import common, api +from jedi import api from pytest import raises def test_preload_modules(): def check_loaded(*modules): # +1 for None module (currently used) - assert len(new) == len(modules) + 1 - for i in modules + ('__builtin__',): - assert [i in k for k in new.keys() if k is not None] + assert len(parser_cache) == len(modules) + 1 + for i in modules: + assert [i in k for k in parser_cache.keys() if k is not None] from jedi import cache temp_cache, cache.parser_cache = cache.parser_cache, {} - new = cache.parser_cache - with common.ignored(KeyError): # performance of tests -> no reload - new['__builtin__'] = temp_cache['__builtin__'] + parser_cache = cache.parser_cache - api.preload_module('datetime') + api.preload_module('sys') + check_loaded() # compiled (c_builtin) modules shouldn't be in the cache. api.preload_module('json', 'token') check_loaded('json', 'token') @@ -29,6 +28,7 @@ def test_preload_modules(): def test_empty_script(): assert api.Script('') + def test_line_number_errors(): """ Script should raise a ValueError if line/column numbers are not in a From 2bde6cde0806caeb93df30082b3a5acd350ebc79 Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Sun, 12 Jan 2014 18:07:58 +0100 Subject: [PATCH 43/52] changing completion of python file objects --- jedi/debug.py | 8 +++--- jedi/evaluate/compiled/__init__.py | 9 ++----- jedi/evaluate/compiled/fake.py | 31 +++++++++++++++--------- jedi/evaluate/compiled/fake/_io.pym | 2 +- jedi/evaluate/compiled/fake/builtins.pym | 9 +++++++ 5 files changed, 36 insertions(+), 23 deletions(-) diff --git a/jedi/debug.py b/jedi/debug.py index 85183892..6ab6ff6b 100644 --- a/jedi/debug.py +++ b/jedi/debug.py @@ -4,13 +4,13 @@ import os import time try: - if not os.name == 'nt': + if os.name == 'nt': + # does not work on Windows, as pyreadline and colorama interfere + raise ImportError + else: # Use colorama for nicer console output. from colorama import Fore, init init() - # does not work on Windows, as pyreadline and colorama interfere - else: - raise ImportError except ImportError: class Fore(object): RED = '' diff --git a/jedi/evaluate/compiled/__init__.py b/jedi/evaluate/compiled/__init__.py index cddf675f..b4fa3daa 100644 --- a/jedi/evaluate/compiled/__init__.py +++ b/jedi/evaluate/compiled/__init__.py @@ -6,7 +6,7 @@ import re import sys import os -from jedi._compatibility import builtins as _builtins, is_py3k, exec_function +from jedi._compatibility import builtins as _builtins, exec_function from jedi import debug from jedi.parser.representation import Base from jedi.cache import underscore_memoization @@ -22,6 +22,7 @@ class PyObject(Base): # comply with the parser start_pos = 0, 0 asserts = [] + path = None # modules have this attribute - set it to None. def __init__(self, obj, parent=None): self.obj = obj @@ -183,12 +184,6 @@ docstr_defaults = { 'string': 'str', } -if is_py3k: - #docstr_defaults['file object'] = 'import io; return io.TextIOWrapper()' - pass # TODO reenable -else: - docstr_defaults['file object'] = 'file' - def _parse_function_doc(doc): """ diff --git a/jedi/evaluate/compiled/fake.py b/jedi/evaluate/compiled/fake.py index 73734e5c..1c29ec48 100644 --- a/jedi/evaluate/compiled/fake.py +++ b/jedi/evaluate/compiled/fake.py @@ -11,6 +11,7 @@ import inspect from jedi._compatibility import is_py3k, builtins from jedi.parser import Parser from jedi.parser.representation import Class +from jedi.evaluate.helpers import FakeName modules = {} @@ -76,16 +77,24 @@ def _load_faked_module(module): except IOError: return module = Parser(source, module_name).module + if module_name == 'builtins' and not is_py3k: + # There are two implementations of `open` for either python 2/3. + # -> Rename the python2 version. + open_func = search_scope(module, 'open') + open_func.name = FakeName('open_python3') + open_func = search_scope(module, 'open_python2') + open_func.name = FakeName('open') modules[module_name] = module return module -def _faked(module, obj, name=None): - def from_scope(scope, obj_name): - for s in scope.subscopes: - if str(s.name) == obj_name: - return s +def search_scope(scope, obj_name): + for s in scope.subscopes: + if str(s.name) == obj_name: + return s + +def _faked(module, obj, name=None): # Crazy underscore actions to try to escape all the internal madness. obj = obj.__class__ if is_class_instance(obj) else obj if module is None: @@ -110,21 +119,21 @@ def _faked(module, obj, name=None): # for methods. if name is None: if inspect.isbuiltin(obj): - return from_scope(faked_mod, obj.__name__) + return search_scope(faked_mod, obj.__name__) elif not inspect.isclass(obj): # object is a method or descriptor - cls = from_scope(faked_mod, obj.__objclass__.__name__) + cls = search_scope(faked_mod, obj.__objclass__.__name__) if cls is None: return - return from_scope(cls, obj.__name__) + return search_scope(cls, obj.__name__) else: if obj == module: - return from_scope(faked_mod, name) + return search_scope(faked_mod, name) else: - cls = from_scope(faked_mod, obj.__name__) + cls = search_scope(faked_mod, obj.__name__) if cls is None: return - return from_scope(cls, name) + return search_scope(cls, name) def get_faked(*args, **kwargs): diff --git a/jedi/evaluate/compiled/fake/_io.pym b/jedi/evaluate/compiled/fake/_io.pym index 0feec3a2..8225477a 100644 --- a/jedi/evaluate/compiled/fake/_io.pym +++ b/jedi/evaluate/compiled/fake/_io.pym @@ -1,3 +1,3 @@ class TextIOWrapper(): def __next__(self): - return '' + return 'hacked io return' diff --git a/jedi/evaluate/compiled/fake/builtins.pym b/jedi/evaluate/compiled/fake/builtins.pym index 19004012..2ab23b62 100644 --- a/jedi/evaluate/compiled/fake/builtins.pym +++ b/jedi/evaluate/compiled/fake/builtins.pym @@ -45,6 +45,15 @@ class xrange(): return 1 +def open(file, mode='r', buffering=-1, encoding=None, errors=None, newline=None, closefd=True): + import io + return io.TextIOWrapper(file, mode, buffering, encoding, errors, newline, closefd) + + +def open_python2(name, mode=None, buffering=None): + return file(name, mode, buffering) + + #-------------------------------------------------------- # descriptors #-------------------------------------------------------- From 860aa501924e7e22471580d7bd4145c83498aad8 Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Sun, 12 Jan 2014 18:17:00 +0100 Subject: [PATCH 44/52] renamed fake/_io.pym to io.pym and fixed some other 'fake' issues --- jedi/evaluate/compiled/fake.py | 6 ++++-- jedi/evaluate/compiled/fake/{_io.pym => io.pym} | 0 2 files changed, 4 insertions(+), 2 deletions(-) rename jedi/evaluate/compiled/fake/{_io.pym => io.pym} (100%) diff --git a/jedi/evaluate/compiled/fake.py b/jedi/evaluate/compiled/fake.py index 1c29ec48..fcad83ae 100644 --- a/jedi/evaluate/compiled/fake.py +++ b/jedi/evaluate/compiled/fake.py @@ -75,16 +75,18 @@ def _load_faked_module(module): with open(os.path.join(path, 'fake', module_name) + '.pym') as f: source = f.read() except IOError: + modules[module_name] = None return module = Parser(source, module_name).module + modules[module_name] = module + if module_name == 'builtins' and not is_py3k: # There are two implementations of `open` for either python 2/3. - # -> Rename the python2 version. + # -> Rename the python2 version (`look at fake/builtins.pym`). open_func = search_scope(module, 'open') open_func.name = FakeName('open_python3') open_func = search_scope(module, 'open_python2') open_func.name = FakeName('open') - modules[module_name] = module return module diff --git a/jedi/evaluate/compiled/fake/_io.pym b/jedi/evaluate/compiled/fake/io.pym similarity index 100% rename from jedi/evaluate/compiled/fake/_io.pym rename to jedi/evaluate/compiled/fake/io.pym From dfb494b9c471bcb9582a5f5a7a29ec1ee9602442 Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Sun, 12 Jan 2014 18:22:33 +0100 Subject: [PATCH 45/52] finally able to delete the old builtin stuff --- jedi/evaluate/builtin.py | 455 --------------------------------------- test/test_builtin.py | 13 -- test/test_compiled.py | 9 + 3 files changed, 9 insertions(+), 468 deletions(-) delete mode 100644 jedi/evaluate/builtin.py delete mode 100644 test/test_builtin.py diff --git a/jedi/evaluate/builtin.py b/jedi/evaluate/builtin.py deleted file mode 100644 index 19e0147c..00000000 --- a/jedi/evaluate/builtin.py +++ /dev/null @@ -1,455 +0,0 @@ -""" -A big part of the Python standard libraries are unfortunately not only written -in Python. The process works like this: - -- ``BuiltinModule`` imports the builtin module (e.g. ``sys``) -- then ``BuiltinModule`` generates code with the docstrings of builtin - functions. -- The :mod:`parsing` parser processes the generated code. - -This is possible, because many builtin functions supply docstrings, for example -the method ``list.index`` has the following attribute ``__doc__``: - - L.index(value, [start, [stop]]) -> integer -- return first index of value. - Raises ValueError if the value is not present. - -`PEP 257 `_ -teaches how docstrings should look like for C functions. - -Additionally there's a ``Builtin`` instance in this module, to make it -possible to access functions like ``list`` and ``int`` directly, the same way -|jedi| access other functions. -""" - -from jedi._compatibility import exec_function, is_py3k - -import re -import sys -import os -if is_py3k: - import io -import types -import inspect - -from jedi import common -from jedi import debug -from jedi.parser import Parser -from jedi.parser import fast -from jedi.evaluate.sys_path import get_sys_path -from jedi import cache - - -class BuiltinModule(object): - """ - This module is a parser for all builtin modules, which are programmed in - C/C++. It should also work on third party modules. - It can be instantiated with either a path or a name of the module. The path - is important for third party modules. - - :param name: The name of the module. - :param path: The path of the module. - :param sys_path: The sys.path, which is can be customizable. - """ - - map_types = { - 'floating point number': '0.0', - 'string': '""', - 'str': '""', - 'character': '"a"', - 'integer': '0', - 'int': '0', - 'dictionary': '{}', - 'list': '[]', - 'file object': 'file("")', - # TODO things like dbg: ('not working', 'tuple of integers') - } - - if is_py3k: - map_types['file object'] = 'import io; return io.TextIOWrapper()' - - def __init__(self, path=None, name=None, sys_path=None): - if sys_path is None: - sys_path = get_sys_path() - self.sys_path = list(sys_path) - - if not name: - name = os.path.basename(path) - name = name.rpartition('.')[0] # cut file type (normally .so) - self.name = name - - self.path = path and os.path.abspath(path) - - @property - @cache.underscore_memoization - def parser(self): - """ get the parser lazy """ - return cache.load_parser(self.path, self.name) or self._load_module() - - def _load_module(self): - source = _generate_code(self.module, self._load_mixins()) - p = self.path or self.name - p = fast.FastParser(source, p) - cache.save_parser(self.path, self.name, p) - return p - - @property - @cache.underscore_memoization - def module(self): - """get module also lazy""" - def load_module(name, path): - if path: - self.sys_path.insert(0, path) - - temp, sys.path = sys.path, self.sys_path - content = {} - try: - exec_function('import %s as module' % name, content) - module = content['module'] - except AttributeError: - # use sys.modules, because you cannot access some modules - # directly. -> #59 - module = sys.modules[name] - sys.path = temp - - if path: - self.sys_path.pop(0) - return module - - # module might already be defined - path = self.path - name = self.name - if self.path: - dot_path = [] - p = self.path - # search for the builtin with the correct path - while p and p not in sys.path: - p, sep, mod = p.rpartition(os.path.sep) - dot_path.append(mod.partition('.')[0]) - if p: - name = ".".join(reversed(dot_path)) - path = p - else: - path = os.path.dirname(self.path) - return load_module(name, path) - - def _load_mixins(self): - """ - Load functions that are mixed in to the standard library. - E.g. builtins are written in C (binaries), but my autocompletion only - understands Python code. By mixing in Python code, the autocompletion - should work much better for builtins. - """ - regex = r'^(def|class)\s+([\w\d]+)' - - def process_code(code, depth=0): - funcs = {} - matches = list(re.finditer(regex, code, re.MULTILINE)) - positions = [m.start() for m in matches] - for i, pos in enumerate(positions): - try: - code_block = code[pos:positions[i + 1]] - except IndexError: - code_block = code[pos:len(code)] - structure_name = matches[i].group(1) - name = matches[i].group(2) - if structure_name == 'def': - funcs[name] = code_block - elif structure_name == 'class': - if depth > 0: - raise NotImplementedError() - - # remove class line - c = re.sub(r'^[^\n]+', '', code_block) - # remove whitespace - c = re.compile(r'^[ ]{4}', re.MULTILINE).sub('', c) - - funcs[name] = process_code(c) - else: - raise NotImplementedError() - return funcs - - name = self.name - # sometimes there are stupid endings like `_sqlite3.cpython-32mu` - name = re.sub(r'\..*', '', name) - - if name == '__builtin__' and not is_py3k: - name = 'builtins' - path = os.path.dirname(os.path.abspath(__file__)) - try: - with open(os.path.join(path, 'mixin', name) + '.pym') as f: - s = f.read() - except IOError: - return {} - else: - mixin_dct = process_code(s) - if is_py3k and self.name == Builtin.name: - # in the case of Py3k xrange is now range - mixin_dct['range'] = mixin_dct['xrange'] - return mixin_dct - - -def _generate_code(scope, mixin_funcs={}, depth=0): - """ - Generate a string, which uses python syntax as an input to the Parser. - """ - def get_doc(obj, indent=False): - doc = inspect.getdoc(obj) - if doc: - doc = ('r"""\n%s\n"""\n' % doc) - if indent: - doc = common.indent_block(doc) - return doc - return '' - - def is_in_base_classes(cls, name, comparison): - """ Base classes may contain the exact same object """ - if name in mixin_funcs: - return False - try: - mro = cls.mro() - except TypeError: - # this happens, if cls == type - return False - for base in mro[1:]: - try: - attr = getattr(base, name) - except AttributeError: - continue - if attr == comparison: - return True - return False - - def get_scope_objects(names): - """ - Looks for the names defined with dir() in an objects and divides - them into different object types. - """ - classes = {} - funcs = {} - stmts = {} - members = {} - for n in names: - try: - # this has a builtin_function_or_method - exe = getattr(scope, n) - except AttributeError: - # happens e.g. in properties of - # PyQt4.QtGui.QStyleOptionComboBox.currentText - # -> just set it to None - members[n] = None - else: - if inspect.isclass(scope): - if is_in_base_classes(scope, n, exe): - continue - if inspect.isbuiltin(exe) or inspect.ismethod(exe) \ - or inspect.ismethoddescriptor(exe): - funcs[n] = exe - elif inspect.isclass(exe) or inspect.ismodule(exe): - classes[n] = exe - elif inspect.ismemberdescriptor(exe): - members[n] = exe - else: - stmts[n] = exe - return classes, funcs, stmts, members - - code = '' - if inspect.ismodule(scope): # generate comment where the code's from. - try: - path = scope.__file__ - except AttributeError: - path = '?' - code += '# Generated module %s from %s\n' % (scope.__name__, path) - - code += get_doc(scope) - - # Remove some magic vars, (TODO why?) - names = set(dir(scope)) - set(['__file__', '__name__', '__doc__', - '__path__', '__package__']) - - classes, funcs, stmts, members = get_scope_objects(names) - - # classes - for name, cl in classes.items(): - bases = (c.__name__ for c in cl.__bases__) if inspect.isclass(cl) \ - else [] - code += 'class %s(%s):\n' % (name, ','.join(bases)) - if depth == 0: - try: - mixin = mixin_funcs[name] - except KeyError: - mixin = {} - cl_code = _generate_code(cl, mixin, depth + 1) - code += common.indent_block(cl_code) - code += '\n' - - # functions - for name, func in funcs.items(): - params, ret = _parse_function_doc(func) - if depth > 0: - params = 'self, ' + params - doc_str = get_doc(func, indent=True) - try: - mixin = mixin_funcs[name] - except KeyError: - # normal code generation - code += 'def %s(%s):\n' % (name, params) - code += doc_str - code += common.indent_block('%s\n\n' % ret) - else: - # generation of code with mixins - # the parser only supports basic functions with a newline after - # the double dots - # find doc_str place - try: - pos = re.search(r'\):\s*\n', mixin).end() - except TypeError: - # pypy uses a different reversed builtin - if name == 'reversed': - mixin = 'def reversed(sequence):\n' \ - ' for i in self.__sequence: yield i' - pos = 24 - else: - debug.warning('mixin trouble in pypy: %s', name) - raise - if pos is None: - raise Exception("Builtin function not parsed correctly") - code += mixin[:pos] + doc_str + mixin[pos:] - - # class members (functions) properties? - for name, func in members.items(): - # recursion problem in properties TODO remove - if name in ['fget', 'fset', 'fdel']: - continue - ret = 'pass' - code += '@property\ndef %s(self):\n' % (name) - code += common.indent_block(get_doc(func) + '%s\n\n' % ret) - - # variables - for name, value in stmts.items(): - if is_py3k: - file_type = io.TextIOWrapper - else: - file_type = types.FileType - if isinstance(value, file_type): - value = 'open()' - elif name == 'None': - value = '' - elif type(value).__name__ in ['int', 'bool', 'float', - 'dict', 'list', 'tuple']: - value = repr(value) - else: - # get the type, if the type is not simple. - mod = type(value).__module__ - value = type(value).__name__ + '()' - if mod != '__builtin__': - value = '%s.%s' % (mod, value) - code += '%s = %s\n' % (name, value) - - return code - - -def _parse_function_doc(func): - """ - Takes a function and returns the params and return value as a tuple. - This is nothing more than a docstring parser. - """ - # TODO: things like utime(path, (atime, mtime)) and a(b [, b]) -> None - doc = inspect.getdoc(func) - - if doc is None: - return '', 'pass' - - # get full string, parse round parentheses: def func(a, (b,c)) - try: - # unbound methods such as pyqtSignals have no __name__ - if not hasattr(func, "__name__"): - return '', 'pass' - count = 0 - debug.dbg(func, func.__name__, doc) - start = doc.index('(') - for i, s in enumerate(doc[start:]): - if s == '(': - count += 1 - elif s == ')': - count -= 1 - if count == 0: - end = start + i - break - param_str = doc[start + 1:end] - except (ValueError, UnboundLocalError): - # ValueError for doc.index - # UnboundLocalError for undefined end in last line - debug.dbg('no brackets found - no param') - end = 0 - param_str = '' - else: - # remove square brackets, that show an optional param ( = None) - def change_options(m): - args = m.group(1).split(',') - for i, a in enumerate(args): - if a and '=' not in a: - args[i] += '=None' - return ','.join(args) - - while True: - param_str, changes = re.subn(r' ?\[([^\[\]]+)\]', - change_options, param_str) - if changes == 0: - break - param_str = param_str.replace('-', '_') # see: isinstance.__doc__ - - # parse return value - r = re.search('-[>-]* ', doc[end:end + 7]) - if r is None: - ret = '' - else: - index = end + r.end() - # get result type, which can contain newlines - pattern = re.compile(r'(,\n|[^\n-])+') - ret_str = pattern.match(doc, index).group(0).strip() - # New object -> object() - ret_str = re.sub(r'[nN]ew (.*)', r'\1()', ret_str) - - ret = BuiltinModule.map_types.get(ret_str, ret_str) - if ret == ret_str and ret not in ['None', 'object', 'tuple', 'set']: - debug.dbg('not working', ret_str) - - ret = ('return ' if 'return' not in ret else '') + ret - return param_str, ret - - -class Builtin(object): - """ The builtin scope / module """ - # Python 3 compatibility - if is_py3k: - name = 'builtins' - else: - name = '__builtin__' - - @property - @cache.underscore_memoization - def builtin(self): - return BuiltinModule(name=self.name) - - @property - def scope(self): - return self.builtin.parser.module - - def magic_function_scope(self, evaluator): - try: - return self._magic_function_scope - except AttributeError: - # depth = 1 because this is not a module - class Container(object): - FunctionType = types.FunctionType - source = _generate_code(Container, depth=0) - parser = Parser(source, None) - module = parser.module - module.parent = self.scope - typ = evaluator.follow_path(iter(['FunctionType']), [module], module) - - s = self._magic_function_scope = typ.pop() - return s - - -Builtin = Builtin() diff --git a/test/test_builtin.py b/test/test_builtin.py deleted file mode 100644 index b9513c30..00000000 --- a/test/test_builtin.py +++ /dev/null @@ -1,13 +0,0 @@ -from jedi.evaluate import builtin - - -def test_parse_function_doc_illegal_docstr(): - - def test_func(a): - """ - test_func(o - - doesn't have a closing bracket. - """ - - assert ('', '') == builtin._parse_function_doc(test_func) diff --git a/test/test_compiled.py b/test/test_compiled.py index 88eb8ba9..bb5aa1c2 100644 --- a/test/test_compiled.py +++ b/test/test_compiled.py @@ -21,3 +21,12 @@ def test_fake_loading(): string = compiled.builtin.get_subscope_by_name('str') from_name = compiled._create_from_name(compiled.builtin, string, '__init__') assert isinstance(from_name, Function) + + +def test_parse_function_doc_illegal_docstr(): + docstr = """ + test_func(o + + doesn't have a closing bracket. + """ + assert ('', '') == compiled._parse_function_doc(docstr) From 4b319ad8176188d72835977de894d3d88c588378 Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Sun, 12 Jan 2014 19:22:31 +0100 Subject: [PATCH 46/52] fix a docstr issue --- jedi/evaluate/compiled/fake.py | 11 +++++++---- test/test_compiled.py | 4 ++++ 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/jedi/evaluate/compiled/fake.py b/jedi/evaluate/compiled/fake.py index fcad83ae..dd6054d9 100644 --- a/jedi/evaluate/compiled/fake.py +++ b/jedi/evaluate/compiled/fake.py @@ -96,7 +96,7 @@ def search_scope(scope, obj_name): return s -def _faked(module, obj, name=None): +def _faked(module, obj, name): # Crazy underscore actions to try to escape all the internal madness. obj = obj.__class__ if is_class_instance(obj) else obj if module is None: @@ -138,9 +138,12 @@ def _faked(module, obj, name=None): return search_scope(cls, name) -def get_faked(*args, **kwargs): - result = _faked(*args, **kwargs) - if not isinstance(result, Class): +def get_faked(module, obj, name=None): + result = _faked(module, obj, name) + if not isinstance(result, Class) and result is not None: + # Set the docstr which was previously not set (faked modules don't + # contain it). + result.docstr = obj.__doc__ return result diff --git a/test/test_compiled.py b/test/test_compiled.py index bb5aa1c2..c358fef6 100644 --- a/test/test_compiled.py +++ b/test/test_compiled.py @@ -23,6 +23,10 @@ def test_fake_loading(): assert isinstance(from_name, Function) +def test_fake_docstr(): + assert compiled.create(next).docstr == next.__doc__ + + def test_parse_function_doc_illegal_docstr(): docstr = """ test_func(o From b2d99be3ee536ece2e4bc67c9ac30aa172d04c0c Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Sun, 12 Jan 2014 19:23:37 +0100 Subject: [PATCH 47/52] delete old builtin code --- jedi/evaluate/compiled/fake.py | 47 ---------------------------------- 1 file changed, 47 deletions(-) diff --git a/jedi/evaluate/compiled/fake.py b/jedi/evaluate/compiled/fake.py index dd6054d9..a1eef5ad 100644 --- a/jedi/evaluate/compiled/fake.py +++ b/jedi/evaluate/compiled/fake.py @@ -4,7 +4,6 @@ written in C (binaries), but my autocompletion only understands Python code. By mixing in Python code, the autocompletion should work much better for builtins. """ -import re import os import inspect @@ -16,52 +15,6 @@ from jedi.evaluate.helpers import FakeName modules = {} -def _load_fakes(module_name): - regex = r'^(def|class)\s+([\w\d]+)' - - def process_code(code, depth=0): - funcs = {} - matches = list(re.finditer(regex, code, re.MULTILINE)) - positions = [m.start() for m in matches] - for i, pos in enumerate(positions): - try: - code_block = code[pos:positions[i + 1]] - except IndexError: - code_block = code[pos:len(code)] - structure_name = matches[i].group(1) - name = matches[i].group(2) - if structure_name == 'def': - funcs[name] = code_block - elif structure_name == 'class': - if depth > 0: - raise NotImplementedError() - - # remove class line - c = re.sub(r'^[^\n]+', '', code_block) - # remove whitespace - c = re.compile(r'^[ ]{4}', re.MULTILINE).sub('', c) - - funcs[name] = process_code(c) - else: - raise NotImplementedError() - return funcs - - if module_name == '__builtin__' and not is_py3k: - module_name = 'builtins' - path = os.path.dirname(os.path.abspath(__file__)) - try: - with open(os.path.join(path, 'mixin', module_name) + '.pym') as f: - s = f.read() - except IOError: - return {} - else: - mixin_dct = process_code(s) - if is_py3k and module_name == 'builtins': - # in the case of Py3k xrange is now range - mixin_dct['range'] = mixin_dct['xrange'] - return mixin_dct - - def _load_faked_module(module): module_name = module.__name__ if module_name == '__builtin__' and not is_py3k: From c602dc1c40f53b6fb30f4d25f90983b0fa183f9a Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Mon, 13 Jan 2014 01:57:26 +0100 Subject: [PATCH 48/52] modules like PyQt4.QtGui are now importable, because the import works again. used a code snippet from the old builtin plugin --- jedi/evaluate/compiled/__init__.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/jedi/evaluate/compiled/__init__.py b/jedi/evaluate/compiled/__init__.py index b4fa3daa..d494571c 100644 --- a/jedi/evaluate/compiled/__init__.py +++ b/jedi/evaluate/compiled/__init__.py @@ -14,10 +14,6 @@ from jedi.evaluate.sys_path import get_sys_path from . import fake -# TODO -# unbound methods such as pyqtSignals have no __name__ -# if not hasattr(func, "__name__"): - class PyObject(Base): # comply with the parser start_pos = 0, 0 @@ -158,6 +154,20 @@ def load_module(path, name): # sometimes there are endings like `_sqlite3.cpython-32mu` name = re.sub(r'\..*', '', name) + if path: + dot_path = [] + p = path + # if path is not in sys.path, we need to make a well defined import + # like `from numpy.core import umath.` + while p and p not in sys.path: + p, sep, mod = p.rpartition(os.path.sep) + dot_path.insert(0, mod.partition('.')[0]) + if p: + name = ".".join(dot_path) + path = p + else: + path = os.path.dirname(path) + sys_path = get_sys_path() if path: sys_path.insert(0, path) @@ -172,7 +182,6 @@ def load_module(path, name): # directly. -> github issue #59 module = sys.modules[name] sys.path = temp - return PyObject(module) From a96a2baf5b638b38536e3836e64e10377d0de8a5 Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Mon, 13 Jan 2014 02:30:10 +0100 Subject: [PATCH 49/52] fix an issue with missing '__class__' methods e.g. in numpy --- jedi/api/classes.py | 2 +- jedi/evaluate/compiled/__init__.py | 8 +++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/jedi/api/classes.py b/jedi/api/classes.py index c3f5d0b1..8549e121 100644 --- a/jedi/api/classes.py +++ b/jedi/api/classes.py @@ -237,7 +237,7 @@ class BaseDefinition(object): """ try: - return self._definition.doc + return self._definition.doc or '' # Always a String, never None. except AttributeError: return self.raw_doc diff --git a/jedi/evaluate/compiled/__init__.py b/jedi/evaluate/compiled/__init__.py index d494571c..fa65e660 100644 --- a/jedi/evaluate/compiled/__init__.py +++ b/jedi/evaluate/compiled/__init__.py @@ -56,7 +56,13 @@ class PyObject(Base): def _cls(self): # Ensures that a PyObject is returned that is not an instance (like list) if fake.is_class_instance(self.obj): - return PyObject(self.obj.__class__, self.parent) + try: + c = self.obj.__class__ + except AttributeError: + # happens with numpy.core.umath._UFUNC_API (you get it + # automatically by doing `import numpy`. + c = type(None) + return PyObject(c, self.parent) return self def get_defined_names(self): From 947890834673ffb29943f8f35843f82de2cc34c5 Mon Sep 17 00:00:00 2001 From: blink1073 Date: Sun, 12 Jan 2014 19:57:02 -0600 Subject: [PATCH 50/52] Add a module to test the memory usage with large libaries --- scripts/memory_check.py | 53 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 scripts/memory_check.py diff --git a/scripts/memory_check.py b/scripts/memory_check.py new file mode 100644 index 00000000..e77c9dfd --- /dev/null +++ b/scripts/memory_check.py @@ -0,0 +1,53 @@ +""" +This is a convenience script to test the speed and memory usage of Jedi with +large libraries. + +Each library is preloaded by jedi, recording the time and memory consumed by +each operation. + +You can provide additional libraries via command line arguments. + +Note: This requires the psutil library, available on PyPI. +""" +import time +import sys +import psutil +import jedi + + +def used_memory(): + """Return the total MB of System Memory in use.""" + return psutil.virtual_memory().used / 2**20 + + +def profile_preload(mod): + """Preload a module into Jedi, recording time and memory used.""" + base = used_memory() + t0 = time.time() + jedi.preload_module(mod) + elapsed = time.time() - t0 + used = used_memory() - base + return elapsed, used + + +def main(mods): + """Preload the modules, and print the time and memory used.""" + t0 = time.time() + baseline = used_memory() + print('Time (s) | Mem (MB) | Package') + print('------------------------------') + for mod in mods: + elapsed, used = profile_preload(mod) + if used > 0: + print('%8.1f | %8d | %s' % (elapsed, used, mod)) + print('------------------------------') + elapsed = time.time() - t0 + used = used_memory() - baseline + print('%8.1f | %8d | %s' % (elapsed, used, 'Total')) + + +if __name__ == '__main__': + mods = ['re', 'numpy', 'scipy', 'scipy.sparse', 'scipy.stats', + 'wx', 'decimal', 'PyQt4.QtGui', 'PySide.QtGui', 'Tkinter'] + mods += sys.argv[1:] + main(mods) From e56a0cf544f89d73b32fe3a8bac33c267c045880 Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Mon, 13 Jan 2014 02:58:10 +0100 Subject: [PATCH 51/52] docstring was sometimes empty for faked modules --- jedi/evaluate/compiled/fake.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jedi/evaluate/compiled/fake.py b/jedi/evaluate/compiled/fake.py index a1eef5ad..345efe60 100644 --- a/jedi/evaluate/compiled/fake.py +++ b/jedi/evaluate/compiled/fake.py @@ -51,7 +51,6 @@ def search_scope(scope, obj_name): def _faked(module, obj, name): # Crazy underscore actions to try to escape all the internal madness. - obj = obj.__class__ if is_class_instance(obj) else obj if module is None: try: module = obj.__objclass__ @@ -92,11 +91,12 @@ def _faked(module, obj, name): def get_faked(module, obj, name=None): + obj = obj.__class__ if is_class_instance(obj) else obj result = _faked(module, obj, name) if not isinstance(result, Class) and result is not None: # Set the docstr which was previously not set (faked modules don't # contain it). - result.docstr = obj.__doc__ + result.docstr = obj.__doc__ or '' return result From 6b3ebe50d83676abe7216eca84c93d8792506fba Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Mon, 13 Jan 2014 03:18:17 +0100 Subject: [PATCH 52/52] scripts folder doesn't need testing - just ignore it --- pytest.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pytest.ini b/pytest.ini index 21f00ce2..fa132557 100644 --- a/pytest.ini +++ b/pytest.ini @@ -2,7 +2,7 @@ addopts = --doctest-modules # Ignore broken files in blackbox test directories -norecursedirs = .* docs completion refactor absolute_import namespace_package +norecursedirs = .* docs completion refactor absolute_import namespace_package scripts # Activate `clean_jedi_cache` fixture for all tests. This should be # fine as long as we are using `clean_jedi_cache` as a session scoped