Merge pull request #363 from davidhalter/builtin

Fixing Memory Issues in Compiled Modules

jedi/api/__init__.py

@@ -27,7 +27,7 @@ from jedi import keywords
 from jedi.api import classes
 from jedi.evaluate import Evaluator, filter_private_variable
 from jedi.evaluate import representation as er
-from jedi.evaluate import builtin
+from jedi.evaluate import compiled
 from jedi.evaluate import imports
 from jedi.evaluate import helpers
 
@@ -137,20 +137,19 @@ class Script(object):
         path, dot, like = self._get_completion_parts()
 
         user_stmt = self._user_stmt(True)
-        bs = builtin.Builtin.scope
-        completions = get_completions(user_stmt, bs)
+        b = compiled.builtin
+        completions = get_completions(user_stmt, b)
 
         if not dot:
             # add named params
             for call_def in self.call_signatures():
-                if not call_def.module.is_builtin():
+                if not isinstance(call_def.module, compiled.PyObject):
                     for p in call_def.params:
                         completions.append((p.get_name(), p))
 
             if not path and not isinstance(user_stmt, pr.Import):
                 # add keywords
-                completions += ((k, bs) for k in keywords.keyword_names(
-                    all=True))
+                completions += ((k, b) for k in keywords.keyword_names(all=True))
 
         needs_dot = not dot and path
 
@@ -193,7 +192,7 @@ class Script(object):
             debug.dbg('possible scopes', scopes)
             for s in scopes:
                 if s.isinstance(er.Function):
-                    names = s.get_magic_method_names()
+                    names = s.get_magic_function_names()
                 else:
                     if isinstance(s, imports.ImportPath):
                         under = like + self._user_context.get_path_after_cursor()
@@ -519,7 +518,8 @@ class Script(object):
         debug.speed('func_call followed')
 
         return [classes.CallDef(o, index, call) for o in origins
-                if o.isinstance(er.Function, er.Instance, er.Class)]
+                if o.isinstance(er.Function, er.Instance, er.Class)
+                or isinstance(o, compiled.PyObject) and o.type() != 'module']
 
     def _func_call_and_param_index(self):
         debug.speed('func_call start')
@@ -563,11 +563,10 @@ class Script(object):
         match = re.match(r'^(.*?)(\.|)(\w?[\w\d]*)$', path, flags=re.S)
         return match.groups()
 
-    @staticmethod
-    def _sorted_defs(d):
+    def _sorted_defs(self, d):
         # Note: `or ''` below is required because `module_path` could be
         #       None and you can't compare None and str in Python 3.
-        return sorted(d, key=lambda x: (x.module_path or '', x.line, x.column))
+        return sorted(d, key=lambda x: (x.module_path or '', x.line or 0, x.column or 0))
 
 
 class Interpreter(Script):

jedi/api/classes.py

@@ -14,6 +14,7 @@ from jedi import cache
 from jedi.evaluate import representation as er
 from jedi.evaluate import iterable
 from jedi.evaluate import imports
+from jedi.evaluate import compiled
 from jedi import keywords
 
 
@@ -70,8 +71,11 @@ class BaseDefinition(object):
 
         # generate a path to the definition
         self._module = definition.get_parent_until()
-        self.module_path = self._module.path
-        """Shows the file path of a module. e.g. ``/usr/lib/python2.7/os.py``"""
+        if self.in_builtin_module():
+            self.module_path = None
+        else:
+            self.module_path = self._module.path
+            """Shows the file path of a module. e.g. ``/usr/lib/python2.7/os.py``"""
 
     @property
     def start_pos(self):
@@ -134,8 +138,10 @@ class BaseDefinition(object):
         """
         # generate the type
         stripped = self._definition
-        if isinstance(self._definition, er.InstanceElement):
-            stripped = self._definition.var
+        if isinstance(stripped, compiled.PyObject):
+            return stripped.type()
+        if isinstance(stripped, er.InstanceElement):
+            stripped = stripped.var
         if isinstance(stripped, pr.Name):
             stripped = stripped.parent
         return type(stripped).__name__.lower()
@@ -167,18 +173,17 @@ class BaseDefinition(object):
         The module name.
 
         >>> from jedi import Script
-        >>> source = 'import datetime'
-        >>> script = Script(source, 1, len(source), 'example.py')
+        >>> source = 'import json'
+        >>> script = Script(source, path='example.py')
         >>> d = script.goto_definitions()[0]
         >>> print(d.module_name)                       # doctest: +ELLIPSIS
-        datetime
+        json
         """
         return str(self._module.name)
 
     def in_builtin_module(self):
         """Whether this is a builtin module."""
-        return not (self.module_path is None or
-                    self.module_path.endswith('.py'))
+        return isinstance(self._module, compiled.PyObject)
 
     @property
     def line_nr(self):
@@ -232,7 +237,7 @@ class BaseDefinition(object):
 
         """
         try:
-            return self._definition.doc
+            return self._definition.doc or ''  # Always a String, never None.
         except AttributeError:
             return self.raw_doc
 
@@ -435,7 +440,9 @@ class Definition(BaseDefinition):
         if isinstance(d, er.InstanceElement):
             d = d.var
 
-        if isinstance(d, pr.Name):
+        if isinstance(d, compiled.PyObject):
+            return d.name
+        elif isinstance(d, pr.Name):
             return d.names[-1] if d.names else None
         elif isinstance(d, iterable.Array):
             return unicode(d.type)
@@ -454,7 +461,6 @@ class Definition(BaseDefinition):
                 return d.assignment_details[0][1].values[0][0].name.names[-1]
             except IndexError:
                 return None
-        return None
 
     @property
     def description(self):
@@ -490,7 +496,9 @@ class Definition(BaseDefinition):
         if isinstance(d, pr.Name):
             d = d.parent
 
-        if isinstance(d, iterable.Array):
+        if isinstance(d, compiled.PyObject):
+            d = d.type() + ' ' + d.name
+        elif isinstance(d, iterable.Array):
             d = 'class ' + d.type
         elif isinstance(d, (pr.Class, er.Class, er.Instance)):
             d = 'class ' + unicode(d.name)
@@ -518,12 +526,7 @@ class Definition(BaseDefinition):
         .. todo:: Add full path. This function is should return a
             `module.class.function` path.
         """
-        if self.module_path.endswith('.py') \
-                and not isinstance(self._definition, pr.Module):
-            position = '@%s' % (self.line)
-        else:
-            # is a builtin or module
-            position = ''
+        position = '' if self.in_builtin_module else '@%s' % (self.line)
         return "%s:%s%s" % (self.module_name, self.description, position)
 
     def defined_names(self):

jedi/debug.py

@@ -4,13 +4,13 @@ import os
 import time
 
 try:
-    if not os.name == 'nt':
+    if os.name == 'nt':
+        # does not work on Windows, as pyreadline and colorama interfere
+        raise ImportError
+    else:
         # Use colorama for nicer console output.
         from colorama import Fore, init
         init()
-        # does not work on Windows, as pyreadline and colorama interfere
-    else:
-        raise ImportError
 except ImportError:
     class Fore(object):
         RED = ''
@@ -29,11 +29,24 @@ enable_notice = False
 # callback, interface: level, str
 debug_function = None
 ignored_modules = ['jedi.evaluate.builtin', 'jedi.parser']
+debug_indent = -1
 
 
 def reset_time():
-    global start_time
+    global start_time, debug_indent
     start_time = time.time()
+    debug_indent = -1
+
+
+def increase_indent(func):
+    """Decorator for makin """
+    def wrapper(*args, **kwargs):
+        global debug_indent
+        debug_indent += 1
+        result = func(*args, **kwargs)
+        debug_indent -= 1
+        return result
+    return wrapper
 
 
 def dbg(*args):
@@ -42,18 +55,21 @@ def dbg(*args):
         frm = inspect.stack()[1]
         mod = inspect.getmodule(frm[0])
         if not (mod.__name__ in ignored_modules):
-            debug_function(NOTICE, 'dbg: ' + ', '.join(u(a) for a in args))
+            i = ' ' * debug_indent
+            debug_function(NOTICE, i + 'dbg: ' + ', '.join(u(a) for a in args))
 
 
 def warning(*args):
     if debug_function and enable_warning:
-        debug_function(WARNING, 'warning: ' + ', '.join(u(a) for a in args))
+        i = ' ' * debug_indent
+        debug_function(WARNING, i + 'warning: ' + ', '.join(u(a) for a in args))
 
 
 def speed(name):
     if debug_function and enable_speed:
         now = time.time()
-        debug_function(SPEED, 'speed: ' + '%s %s' % (name, now - start_time))
+        i = ' ' * debug_indent
+        debug_function(SPEED, i + 'speed: ' + '%s %s' % (name, now - start_time))
 
 
 def print_to_stdout(level, str_out):

jedi/evaluate/__init__.py

@@ -76,13 +76,13 @@ from jedi import common
 from jedi.parser import representation as pr
 from jedi import debug
 from jedi.evaluate import representation as er
-from jedi.evaluate import builtin
 from jedi.evaluate import imports
 from jedi.evaluate import recursion
 from jedi.evaluate import iterable
 from jedi.evaluate.cache import memoize_default
 from jedi.evaluate import stdlib
 from jedi.evaluate import finder
+from jedi.evaluate import compiled
 
 
 class Evaluator(object):
@@ -128,7 +128,7 @@ class Evaluator(object):
         true (default).
 
         >>> pairs[2]                                        #doctest: +ELLIPSIS
-        (<Module: ...builtin...>, [<Name: ...>, ...])
+        (<Builtin: ...builtin...>, [<PyName: ...>, ...])
 
         :rtype: [(pr.Scope, [pr.Name])]
         :return: Return an generator that yields a pair of scope and names.
@@ -146,7 +146,9 @@ class Evaluator(object):
             if not (scope != non_flow and scope.isinstance(pr.Class)
                     or scope.isinstance(pr.Flow)
                     or scope.isinstance(er.Instance)
-                    and non_flow.isinstance(er.Function)):
+                    and non_flow.isinstance(er.Function)
+                    or isinstance(scope, compiled.PyObject)
+                    and scope.type() == 'class' and in_func_scope != scope):
                 try:
                     if isinstance(scope, er.Instance):
                         for g in scope.scope_generator():
@@ -173,8 +175,7 @@ class Evaluator(object):
 
             # Add builtins to the global scope.
             if include_builtin:
-                builtin_scope = builtin.Builtin.scope
-                yield builtin_scope, builtin_scope.get_defined_names()
+                yield compiled.builtin, compiled.builtin.get_defined_names()
 
     def find_types(self, scope, name_str, position=None, search_global=False,
                    is_goto=False, resolve_decorator=True):
@@ -194,15 +195,15 @@ class Evaluator(object):
 
     @memoize_default(default=(), evaluator_is_first_arg=True)
     @recursion.recursion_decorator
+    @debug.increase_indent
     def eval_statement(self, stmt, seek_name=None):
         """
-        The starting point of the completion. A statement always owns a call list,
-        which are the calls, that a statement does.
-        In case multiple names are defined in the statement, `seek_name` returns
-        the result for this name.
+        The starting point of the completion. A statement always owns a call
+        list, which are the calls, that a statement does. In case multiple
+        names are defined in the statement, `seek_name` returns the result for
+        this name.
 
         :param stmt: A `pr.Statement`.
-        :param seek_name: A string.
         """
         debug.dbg('eval_statement %s (%s)' % (stmt, seek_name))
         expression_list = stmt.expression_list()
@@ -266,6 +267,8 @@ class Evaluator(object):
                         er.Function, er.Class, er.Instance, iterable.ArrayInstance):
                     result.append(call)
                 # The string tokens are just operations (+, -, etc.)
+                elif isinstance(call, compiled.PyObject):
+                    result.append(call)
                 elif not isinstance(call, (str, unicode)):
                     if isinstance(call, pr.Call) and str(call.name) == 'if':
                         # Ternary operators.
@@ -281,8 +284,8 @@ class Evaluator(object):
                     result += self.eval_call(call)
                 elif call == '*':
                     if [r for r in result if isinstance(r, iterable.Array)
-                       or isinstance(r, er.Instance)
-                       and str(r.name) == 'str']:
+                            or isinstance(r, compiled.PyObject)
+                            and isinstance(r.obj, (str, unicode))]:
                         # if it is an iterable, ignore * operations
                         next(calls_iterator)
         return set(result)
@@ -308,16 +311,12 @@ class Evaluator(object):
         else:
             if isinstance(current, pr.NamePart):
                 # This is the first global lookup.
-                scopes = self.find_types(scope, current, position=position,
-                                         search_global=True)
+                types = self.find_types(scope, current, position=position,
+                                        search_global=True)
             else:
                 # for pr.Literal
-                scopes = self.find_types(builtin.Builtin.scope, current.type_as_string())
-                # Make instances of those number/string objects.
-                scopes = itertools.chain.from_iterable(
-                    self.execute(s, (current.value,)) for s in scopes
-                )
-            types = imports.strip_imports(self, scopes)
+                types = [compiled.create(current.value)]
+            types = imports.strip_imports(self, types)
 
         return self.follow_path(path, types, scope, position=position)
 
@@ -332,8 +331,8 @@ class Evaluator(object):
         results_new = []
         iter_paths = itertools.tee(path, len(types))
 
-        for i, type in enumerate(types):
-            fp = self._follow_path(iter_paths[i], type, call_scope, position=position)
+        for i, typ in enumerate(types):
+            fp = self._follow_path(iter_paths[i], typ, call_scope, position=position)
             if fp is not None:
                 results_new += fp
             else:
@@ -341,7 +340,7 @@ class Evaluator(object):
                 return types
         return results_new
 
-    def _follow_path(self, path, type, scope, position=None):
+    def _follow_path(self, path, typ, scope, position=None):
         """
         Uses a generator and tries to complete the path, e.g.::
 
@@ -355,52 +354,58 @@ class Evaluator(object):
             current = next(path)
         except StopIteration:
             return None
-        debug.dbg('_follow_path: %s in scope %s' % (current, type))
+        debug.dbg('_follow_path: %s in scope %s' % (current, typ))
 
         result = []
         if isinstance(current, pr.Array):
             # This must be an execution, either () or [].
             if current.type == pr.Array.LIST:
-                if hasattr(type, 'get_index_types'):
-                    result = type.get_index_types(current)
+                if hasattr(typ, 'get_index_types'):
+                    result = typ.get_index_types(current)
             elif current.type not in [pr.Array.DICT]:
                 # Scope must be a class or func - make an instance or execution.
-                debug.dbg('exe', type)
-                result = self.execute(type, current)
+                result = self.execute(typ, current)
             else:
                 # Curly braces are not allowed, because they make no sense.
-                debug.warning('strange function call with {}', current, type)
+                debug.warning('strange function call with {}', current, typ)
         else:
             # The function must not be decorated with something else.
-            if type.isinstance(er.Function):
-                type = type.get_magic_method_scope()
+            if typ.isinstance(er.Function):
+                typ = typ.get_magic_function_scope()
             else:
                 # This is the typical lookup while chaining things.
-                if filter_private_variable(type, scope, current):
+                if filter_private_variable(typ, scope, current):
                     return []
-            result = imports.strip_imports(self, self.find_types(type, current,
-                                           position=position))
+            types = self.find_types(typ, current, position=position)
+            result = imports.strip_imports(self, types)
         return self.follow_path(path, set(result), scope, position=position)
 
+    @debug.increase_indent
     def execute(self, obj, params=(), evaluate_generator=False):
         if obj.isinstance(er.Function):
             obj = obj.get_decorated_func()
 
+        debug.dbg('execute:', obj, params)
         try:
             return stdlib.execute(self, obj, params)
         except stdlib.NotInStdLib:
             pass
 
-        if obj.isinstance(er.Class):
+        if obj.isinstance(compiled.PyObject):
+            if obj.is_executable_class():
+                return [er.Instance(self, obj, params)]
+            else:
+                return list(obj.execute_function(self, params))
+        elif obj.isinstance(er.Class):
             # There maybe executions of executions.
             return [er.Instance(self, obj, params)]
         elif isinstance(obj, iterable.Generator):
             return obj.iter_content()
         else:
             stmts = []
-            try:
-                obj.returns  # Test if it is a function
-            except AttributeError:
+            if obj.isinstance(er.Function):
+                stmts = er.FunctionExecution(self, obj, params).get_return_types(evaluate_generator)
+            else:
                 if hasattr(obj, 'execute_subscope_by_name'):
                     try:
                         stmts = obj.execute_subscope_by_name('__call__', params)
@@ -408,10 +413,8 @@ class Evaluator(object):
                         debug.warning("no __call__ func available", obj)
                 else:
                     debug.warning("no execution possible", obj)
-            else:
-                stmts = er.FunctionExecution(self, obj, params).get_return_types(evaluate_generator)
 
-            debug.dbg('execute: %s in %s' % (stmts, self))
+            debug.dbg('execute result: %s in %s' % (stmts, obj))
             return imports.strip_imports(self, stmts)
 
     def goto(self, stmt, call_path=None):
@@ -450,9 +453,14 @@ def filter_private_variable(scope, call_scope, var_name):
     """private variables begin with a double underline `__`"""
     if isinstance(var_name, (str, unicode)) and isinstance(scope, er.Instance)\
             and var_name.startswith('__') and not var_name.endswith('__'):
-        s = call_scope.get_parent_until((pr.Class, er.Instance))
-        if s != scope and s != scope.base.base:
-            return True
+        s = call_scope.get_parent_until((pr.Class, er.Instance, compiled.PyObject))
+        if s != scope:
+            if isinstance(scope.base, compiled.PyObject):
+                if s != scope.base:
+                    return True
+            else:
+                if s != scope.base.base:
+                    return True
     return False
 
 

jedi/evaluate/builtin.py

@@ -1,455 +0,0 @@
-"""
-A big part of the Python standard libraries are unfortunately not only written
-in Python. The process works like this:
-
-- ``BuiltinModule`` imports the builtin module (e.g. ``sys``)
-- then ``BuiltinModule`` generates code with the docstrings of builtin
-  functions.
-- The :mod:`parsing` parser processes the generated code.
-
-This is possible, because many builtin functions supply docstrings, for example
-the method ``list.index`` has the following attribute ``__doc__``:
-
-    L.index(value, [start, [stop]]) -> integer -- return first index of value.
-        Raises ValueError if the value is not present.
-
-`PEP 257 <http://www.python.org/dev/peps/pep-0257/#one-line-docstrings>`_
-teaches how docstrings should look like for C functions.
-
-Additionally there's a ``Builtin``  instance in this module, to make it
-possible to access functions like ``list`` and ``int`` directly, the same way
-|jedi| access other functions.
-"""
-
-from jedi._compatibility import exec_function, is_py3k
-
-import re
-import sys
-import os
-if is_py3k:
-    import io
-import types
-import inspect
-
-from jedi import common
-from jedi import debug
-from jedi.parser import Parser
-from jedi.parser import fast
-from jedi.evaluate.sys_path import get_sys_path
-from jedi import cache
-
-
-class BuiltinModule(object):
-    """
-    This module is a parser for all builtin modules, which are programmed in
-    C/C++. It should also work on third party modules.
-    It can be instantiated with either a path or a name of the module. The path
-    is important for third party modules.
-
-    :param name: The name of the module.
-    :param path: The path of the module.
-    :param sys_path: The sys.path, which is can be customizable.
-    """
-
-    map_types = {
-        'floating point number': '0.0',
-        'string': '""',
-        'str': '""',
-        'character': '"a"',
-        'integer': '0',
-        'int': '0',
-        'dictionary': '{}',
-        'list': '[]',
-        'file object': 'file("")',
-        # TODO things like dbg: ('not working', 'tuple of integers')
-    }
-
-    if is_py3k:
-        map_types['file object'] = 'import io; return io.TextIOWrapper()'
-
-    def __init__(self, path=None, name=None, sys_path=None):
-        if sys_path is None:
-            sys_path = get_sys_path()
-        self.sys_path = list(sys_path)
-
-        if not name:
-            name = os.path.basename(path)
-            name = name.rpartition('.')[0]  # cut file type (normally .so)
-        self.name = name
-
-        self.path = path and os.path.abspath(path)
-
-    @property
-    @cache.underscore_memoization
-    def parser(self):
-        """ get the parser lazy """
-        return cache.load_parser(self.path, self.name) or self._load_module()
-
-    def _load_module(self):
-        source = _generate_code(self.module, self._load_mixins())
-        p = self.path or self.name
-        p = fast.FastParser(source, p)
-        cache.save_parser(self.path, self.name, p)
-        return p
-
-    @property
-    @cache.underscore_memoization
-    def module(self):
-        """get module also lazy"""
-        def load_module(name, path):
-            if path:
-                self.sys_path.insert(0, path)
-
-            temp, sys.path = sys.path, self.sys_path
-            content = {}
-            try:
-                exec_function('import %s as module' % name, content)
-                module = content['module']
-            except AttributeError:
-                # use sys.modules, because you cannot access some modules
-                # directly. -> #59
-                module = sys.modules[name]
-            sys.path = temp
-
-            if path:
-                self.sys_path.pop(0)
-            return module
-
-        # module might already be defined
-        path = self.path
-        name = self.name
-        if self.path:
-            dot_path = []
-            p = self.path
-            # search for the builtin with the correct path
-            while p and p not in sys.path:
-                p, sep, mod = p.rpartition(os.path.sep)
-                dot_path.append(mod.partition('.')[0])
-            if p:
-                name = ".".join(reversed(dot_path))
-                path = p
-            else:
-                path = os.path.dirname(self.path)
-        return load_module(name, path)
-
-    def _load_mixins(self):
-        """
-        Load functions that are mixed in to the standard library.
-        E.g. builtins are written in C (binaries), but my autocompletion only
-        understands Python code. By mixing in Python code, the autocompletion
-        should work much better for builtins.
-        """
-        regex = r'^(def|class)\s+([\w\d]+)'
-
-        def process_code(code, depth=0):
-            funcs = {}
-            matches = list(re.finditer(regex, code, re.MULTILINE))
-            positions = [m.start() for m in matches]
-            for i, pos in enumerate(positions):
-                try:
-                    code_block = code[pos:positions[i + 1]]
-                except IndexError:
-                    code_block = code[pos:len(code)]
-                structure_name = matches[i].group(1)
-                name = matches[i].group(2)
-                if structure_name == 'def':
-                    funcs[name] = code_block
-                elif structure_name == 'class':
-                    if depth > 0:
-                        raise NotImplementedError()
-
-                    # remove class line
-                    c = re.sub(r'^[^\n]+', '', code_block)
-                    # remove whitespace
-                    c = re.compile(r'^[ ]{4}', re.MULTILINE).sub('', c)
-
-                    funcs[name] = process_code(c)
-                else:
-                    raise NotImplementedError()
-            return funcs
-
-        name = self.name
-        # sometimes there are stupid endings like `_sqlite3.cpython-32mu`
-        name = re.sub(r'\..*', '', name)
-
-        if name == '__builtin__' and not is_py3k:
-            name = 'builtins'
-        path = os.path.dirname(os.path.abspath(__file__))
-        try:
-            with open(os.path.join(path, 'mixin', name) + '.pym') as f:
-                s = f.read()
-        except IOError:
-            return {}
-        else:
-            mixin_dct = process_code(s)
-            if is_py3k and self.name == Builtin.name:
-                # in the case of Py3k xrange is now range
-                mixin_dct['range'] = mixin_dct['xrange']
-            return mixin_dct
-
-
-def _generate_code(scope, mixin_funcs={}, depth=0):
-    """
-    Generate a string, which uses python syntax as an input to the Parser.
-    """
-    def get_doc(obj, indent=False):
-        doc = inspect.getdoc(obj)
-        if doc:
-            doc = ('r"""\n%s\n"""\n' % doc)
-            if indent:
-                doc = common.indent_block(doc)
-            return doc
-        return ''
-
-    def is_in_base_classes(cls, name, comparison):
-        """ Base classes may contain the exact same object """
-        if name in mixin_funcs:
-            return False
-        try:
-            mro = cls.mro()
-        except TypeError:
-            # this happens, if cls == type
-            return False
-        for base in mro[1:]:
-            try:
-                attr = getattr(base, name)
-            except AttributeError:
-                continue
-            if attr == comparison:
-                return True
-        return False
-
-    def get_scope_objects(names):
-        """
-        Looks for the names defined with dir() in an objects and divides
-        them into different object types.
-        """
-        classes = {}
-        funcs = {}
-        stmts = {}
-        members = {}
-        for n in names:
-            try:
-                # this has a builtin_function_or_method
-                exe = getattr(scope, n)
-            except AttributeError:
-                # happens e.g. in properties of
-                # PyQt4.QtGui.QStyleOptionComboBox.currentText
-                # -> just set it to None
-                members[n] = None
-            else:
-                if inspect.isclass(scope):
-                    if is_in_base_classes(scope, n, exe):
-                        continue
-                if inspect.isbuiltin(exe) or inspect.ismethod(exe) \
-                        or inspect.ismethoddescriptor(exe):
-                    funcs[n] = exe
-                elif inspect.isclass(exe) or inspect.ismodule(exe):
-                    classes[n] = exe
-                elif inspect.ismemberdescriptor(exe):
-                    members[n] = exe
-                else:
-                    stmts[n] = exe
-        return classes, funcs, stmts, members
-
-    code = ''
-    if inspect.ismodule(scope):  # generate comment where the code's from.
-        try:
-            path = scope.__file__
-        except AttributeError:
-            path = '?'
-        code += '# Generated module %s from %s\n' % (scope.__name__, path)
-
-    code += get_doc(scope)
-
-    # Remove some magic vars, (TODO why?)
-    names = set(dir(scope)) - set(['__file__', '__name__', '__doc__',
-                                   '__path__', '__package__'])
-
-    classes, funcs, stmts, members = get_scope_objects(names)
-
-    # classes
-    for name, cl in classes.items():
-        bases = (c.__name__ for c in cl.__bases__) if inspect.isclass(cl) \
-            else []
-        code += 'class %s(%s):\n' % (name, ','.join(bases))
-        if depth == 0:
-            try:
-                mixin = mixin_funcs[name]
-            except KeyError:
-                mixin = {}
-            cl_code = _generate_code(cl, mixin, depth + 1)
-            code += common.indent_block(cl_code)
-        code += '\n'
-
-    # functions
-    for name, func in funcs.items():
-        params, ret = _parse_function_doc(func)
-        if depth > 0:
-            params = 'self, ' + params
-        doc_str = get_doc(func, indent=True)
-        try:
-            mixin = mixin_funcs[name]
-        except KeyError:
-            # normal code generation
-            code += 'def %s(%s):\n' % (name, params)
-            code += doc_str
-            code += common.indent_block('%s\n\n' % ret)
-        else:
-            # generation of code with mixins
-            # the parser only supports basic functions with a newline after
-            # the double dots
-            # find doc_str place
-            try:
-                pos = re.search(r'\):\s*\n', mixin).end()
-            except TypeError:
-                # pypy uses a different reversed builtin
-                if name == 'reversed':
-                    mixin = 'def reversed(sequence):\n' \
-                            '    for i in self.__sequence: yield i'
-                    pos = 24
-                else:
-                    debug.warning('mixin trouble in pypy: %s', name)
-                    raise
-            if pos is None:
-                raise Exception("Builtin function not parsed correctly")
-            code += mixin[:pos] + doc_str + mixin[pos:]
-
-    # class members (functions) properties?
-    for name, func in members.items():
-        # recursion problem in properties TODO remove
-        if name in ['fget', 'fset', 'fdel']:
-            continue
-        ret = 'pass'
-        code += '@property\ndef %s(self):\n' % (name)
-        code += common.indent_block(get_doc(func) + '%s\n\n' % ret)
-
-    # variables
-    for name, value in stmts.items():
-        if is_py3k:
-            file_type = io.TextIOWrapper
-        else:
-            file_type = types.FileType
-        if isinstance(value, file_type):
-            value = 'open()'
-        elif name == 'None':
-            value = ''
-        elif type(value).__name__ in ['int', 'bool', 'float',
-                                      'dict', 'list', 'tuple']:
-            value = repr(value)
-        else:
-            # get the type, if the type is not simple.
-            mod = type(value).__module__
-            value = type(value).__name__ + '()'
-            if mod != '__builtin__':
-                value = '%s.%s' % (mod, value)
-        code += '%s = %s\n' % (name, value)
-
-    return code
-
-
-def _parse_function_doc(func):
-    """
-    Takes a function and returns the params and return value as a tuple.
-    This is nothing more than a docstring parser.
-    """
-    # TODO: things like utime(path, (atime, mtime)) and a(b [, b]) -> None
-    doc = inspect.getdoc(func)
-
-    if doc is None:
-        return '', 'pass'
-
-    # get full string, parse round parentheses: def func(a, (b,c))
-    try:
-        # unbound methods such as pyqtSignals have no __name__
-        if not hasattr(func, "__name__"):
-            return '', 'pass'
-        count = 0
-        debug.dbg(func, func.__name__, doc)
-        start = doc.index('(')
-        for i, s in enumerate(doc[start:]):
-            if s == '(':
-                count += 1
-            elif s == ')':
-                count -= 1
-            if count == 0:
-                end = start + i
-                break
-        param_str = doc[start + 1:end]
-    except (ValueError, UnboundLocalError):
-        # ValueError for doc.index
-        # UnboundLocalError for undefined end in last line
-        debug.dbg('no brackets found - no param')
-        end = 0
-        param_str = ''
-    else:
-        # remove square brackets, that show an optional param ( = None)
-        def change_options(m):
-            args = m.group(1).split(',')
-            for i, a in enumerate(args):
-                if a and '=' not in a:
-                    args[i] += '=None'
-            return ','.join(args)
-
-        while True:
-            param_str, changes = re.subn(r' ?\[([^\[\]]+)\]',
-                                         change_options, param_str)
-            if changes == 0:
-                break
-    param_str = param_str.replace('-', '_')  # see: isinstance.__doc__
-
-    # parse return value
-    r = re.search('-[>-]* ', doc[end:end + 7])
-    if r is None:
-        ret = ''
-    else:
-        index = end + r.end()
-        # get result type, which can contain newlines
-        pattern = re.compile(r'(,\n|[^\n-])+')
-        ret_str = pattern.match(doc, index).group(0).strip()
-        # New object -> object()
-        ret_str = re.sub(r'[nN]ew (.*)', r'\1()', ret_str)
-
-        ret = BuiltinModule.map_types.get(ret_str, ret_str)
-        if ret == ret_str and ret not in ['None', 'object', 'tuple', 'set']:
-            debug.dbg('not working', ret_str)
-
-        ret = ('return ' if 'return' not in ret else '') + ret
-    return param_str, ret
-
-
-class Builtin(object):
-    """ The builtin scope / module """
-    # Python 3 compatibility
-    if is_py3k:
-        name = 'builtins'
-    else:
-        name = '__builtin__'
-
-    @property
-    @cache.underscore_memoization
-    def builtin(self):
-        return BuiltinModule(name=self.name)
-
-    @property
-    def scope(self):
-        return self.builtin.parser.module
-
-    def magic_function_scope(self, evaluator):
-        try:
-            return self._magic_function_scope
-        except AttributeError:
-            # depth = 1 because this is not a module
-            class Container(object):
-                FunctionType = types.FunctionType
-            source = _generate_code(Container, depth=0)
-            parser = Parser(source, None)
-            module = parser.module
-            module.parent = self.scope
-            typ = evaluator.follow_path(iter(['FunctionType']), [module], module)
-
-            s = self._magic_function_scope = typ.pop()
-            return s
-
-
-Builtin = Builtin()

jedi/evaluate/compiled/__init__.py

@@ -0,0 +1,302 @@
+"""
+Imitate the parser representation.
+"""
+import inspect
+import re
+import sys
+import os
+
+from jedi._compatibility import builtins as _builtins, exec_function
+from jedi import debug
+from jedi.parser.representation import Base
+from jedi.cache import underscore_memoization
+from jedi.evaluate.sys_path import get_sys_path
+from . import fake
+
+
+class PyObject(Base):
+    # comply with the parser
+    start_pos = 0, 0
+    asserts = []
+    path = None  # modules have this attribute - set it to None.
+
+    def __init__(self, obj, parent=None):
+        self.obj = obj
+        self.parent = parent
+        self.doc = inspect.getdoc(obj)
+
+    def __repr__(self):
+        return '<%s: %s>' % (type(self).__name__, self.obj)
+
+    def get_parent_until(self, *args, **kwargs):
+        # compiled modules only use functions and classes/methods (2 levels)
+        return getattr(self.parent, 'parent', self.parent) or self.parent or self
+
+    @underscore_memoization
+    def _parse_function_doc(self):
+        if self.doc is None:
+            return '', ''
+
+        return _parse_function_doc(self.doc)
+
+    def type(self):
+        cls = self._cls().obj
+        if inspect.isclass(cls):
+            return 'class'
+        elif inspect.ismodule(cls):
+            return 'module'
+        elif inspect.isbuiltin(cls) or inspect.ismethod(cls) \
+                or inspect.ismethoddescriptor(cls):
+            return 'def'
+
+    def is_executable_class(self):
+        return inspect.isclass(self.obj)
+
+    @underscore_memoization
+    def _cls(self):
+        # Ensures that a PyObject is returned that is not an instance (like list)
+        if fake.is_class_instance(self.obj):
+            try:
+                c = self.obj.__class__
+            except AttributeError:
+                # happens with numpy.core.umath._UFUNC_API (you get it
+                # automatically by doing `import numpy`.
+                c = type(None)
+            return PyObject(c, self.parent)
+        return self
+
+    def get_defined_names(self):
+        cls = self._cls()
+        for name in dir(cls.obj):
+            yield PyName(cls, name)
+
+    def instance_names(self):
+        # TODO REMOVE (temporary until the Instance method is removed)
+        return self.get_defined_names()
+
+    def get_subscope_by_name(self, name):
+        if name in dir(self._cls().obj):
+            return PyName(self._cls(), name).parent
+        else:
+            raise KeyError("CompiledObject doesn't have an attribute '%s'." % name)
+
+    @property
+    def name(self):
+        # might not exist sometimes (raises AttributeError)
+        return self._cls().obj.__name__
+
+    def execute_function(self, evaluator, params):
+        if self.type() != 'def':
+            return
+        for name in self._parse_function_doc()[1].split():
+            try:
+                bltn_obj = _create_from_name(builtin, builtin, name)
+            except AttributeError:
+                continue
+            else:
+                if isinstance(bltn_obj, PyObject):
+                    yield bltn_obj
+                else:
+                    for result in evaluator.execute(bltn_obj, params):
+                        yield result
+
+    @property
+    @underscore_memoization
+    def subscopes(self):
+        """
+        Returns only the faked scopes - the other ones are not important for
+        internal analysis.
+        """
+        module = self.get_parent_until()
+        faked_subscopes = []
+        for name in dir(self._cls().obj):
+            f = fake.get_faked(module.obj, self.obj, name)
+            if f:
+                f.parent = self
+                faked_subscopes.append(f)
+        return faked_subscopes
+
+    def get_self_attributes(self):
+        return []  # Instance compatibility
+
+    def get_imports(self):
+        return []  # Builtins don't have imports
+
+
+class PyName(object):
+    def __init__(self, obj, name):
+        self._obj = obj
+        self.name = name
+        self.start_pos = 0, 0  # an illegal start_pos, to make sorting easy.
+
+    def get_parent_until(self):
+        return self.parent.get_parent_until()
+
+    def __str__(self):
+        return self.name
+
+    def __repr__(self):
+        return '<%s: (%s).%s>' % (type(self).__name__, self._obj.name, self.name)
+
+    @property
+    @underscore_memoization
+    def parent(self):
+        module = self._obj.get_parent_until()
+        return _create_from_name(module, self._obj, self.name)
+
+    @property
+    def names(self):
+        return [self.name]  # compatibility with parser.representation.Name
+
+    def get_code(self):
+        return self.name
+
+
+def load_module(path, name):
+    if not name:
+        name = os.path.basename(path)
+        name = name.rpartition('.')[0]  # cut file type (normally .so)
+
+    # sometimes there are endings like `_sqlite3.cpython-32mu`
+    name = re.sub(r'\..*', '', name)
+
+    if path:
+        dot_path = []
+        p = path
+        # if path is not in sys.path, we need to make a well defined import
+        # like `from numpy.core import umath.`
+        while p and p not in sys.path:
+            p, sep, mod = p.rpartition(os.path.sep)
+            dot_path.insert(0, mod.partition('.')[0])
+        if p:
+            name = ".".join(dot_path)
+            path = p
+        else:
+            path = os.path.dirname(path)
+
+    sys_path = get_sys_path()
+    if path:
+        sys_path.insert(0, path)
+
+    temp, sys.path = sys.path, sys_path
+    content = {}
+    try:
+        exec_function('import %s as module' % name, content)
+        module = content['module']
+    except AttributeError:
+        # use sys.modules, because you cannot access some modules
+        # directly. -> github issue #59
+        module = sys.modules[name]
+    sys.path = temp
+    return PyObject(module)
+
+
+docstr_defaults = {
+    'floating point number': 'float',
+    'character': 'str',
+    'integer': 'int',
+    'dictionary': 'dict',
+    'string': 'str',
+}
+
+
+def _parse_function_doc(doc):
+    """
+    Takes a function and returns the params and return value as a tuple.
+    This is nothing more than a docstring parser.
+
+    TODO docstrings like utime(path, (atime, mtime)) and a(b [, b]) -> None
+    TODO docstrings like 'tuple of integers'
+    """
+    # parse round parentheses: def func(a, (b,c))
+    try:
+        count = 0
+        start = doc.index('(')
+        for i, s in enumerate(doc[start:]):
+            if s == '(':
+                count += 1
+            elif s == ')':
+                count -= 1
+            if count == 0:
+                end = start + i
+                break
+        param_str = doc[start + 1:end]
+    except (ValueError, UnboundLocalError):
+        # ValueError for doc.index
+        # UnboundLocalError for undefined end in last line
+        debug.dbg('no brackets found - no param')
+        end = 0
+        param_str = ''
+    else:
+        # remove square brackets, that show an optional param ( = None)
+        def change_options(m):
+            args = m.group(1).split(',')
+            for i, a in enumerate(args):
+                if a and '=' not in a:
+                    args[i] += '=None'
+            return ','.join(args)
+
+        while True:
+            param_str, changes = re.subn(r' ?\[([^\[\]]+)\]',
+                                         change_options, param_str)
+            if changes == 0:
+                break
+    param_str = param_str.replace('-', '_')  # see: isinstance.__doc__
+
+    # parse return value
+    r = re.search('-[>-]* ', doc[end:end + 7])
+    if r is None:
+        ret = ''
+    else:
+        index = end + r.end()
+        # get result type, which can contain newlines
+        pattern = re.compile(r'(,\n|[^\n-])+')
+        ret_str = pattern.match(doc, index).group(0).strip()
+        # New object -> object()
+        ret_str = re.sub(r'[nN]ew (.*)', r'\1()', ret_str)
+
+        ret = docstr_defaults.get(ret_str, ret_str)
+
+    return param_str, ret
+
+
+class Builtin(PyObject):
+    def get_defined_names(self):
+        # Filter None, because it's really just a keyword, nobody wants to
+        # access it.
+        return [d for d in super(Builtin, self).get_defined_names() if d.name != 'None']
+
+
+builtin = Builtin(_builtins)
+magic_function_class = PyObject(type(load_module), parent=builtin)
+
+
+def _create_from_name(module, parent, name):
+    faked = fake.get_faked(module.obj, parent.obj, name)
+    # only functions are necessary.
+    if faked is not None:
+        faked.parent = parent
+        return faked
+
+    try:
+        obj = getattr(parent.obj, name)
+    except AttributeError:
+        # happens e.g. in properties of
+        # PyQt4.QtGui.QStyleOptionComboBox.currentText
+        # -> just set it to None
+        obj = None
+    return PyObject(obj, parent)
+
+
+def create(obj, parent=builtin, module=None):
+    """
+    A very weird interface class to this module. The more options provided the
+    more acurate loading compiled objects is.
+    """
+    if not inspect.ismodule(parent):
+        faked = fake.get_faked(module and module.obj, obj)
+        if faked is not None:
+            faked.parent = parent
+            return faked
+
+    return PyObject(obj, parent)

jedi/evaluate/compiled/fake.py

@@ -0,0 +1,107 @@
+"""
+Loads functions that are mixed in to the standard library. E.g. builtins are
+written in C (binaries), but my autocompletion only understands Python code. By
+mixing in Python code, the autocompletion should work much better for builtins.
+"""
+
+import os
+import inspect
+
+from jedi._compatibility import is_py3k, builtins
+from jedi.parser import Parser
+from jedi.parser.representation import Class
+from jedi.evaluate.helpers import FakeName
+
+modules = {}
+
+
+def _load_faked_module(module):
+    module_name = module.__name__
+    if module_name == '__builtin__' and not is_py3k:
+        module_name = 'builtins'
+
+    try:
+        return modules[module_name]
+    except KeyError:
+        path = os.path.dirname(os.path.abspath(__file__))
+        try:
+            with open(os.path.join(path, 'fake', module_name) + '.pym') as f:
+                source = f.read()
+        except IOError:
+            modules[module_name] = None
+            return
+        module = Parser(source, module_name).module
+        modules[module_name] = module
+
+        if module_name == 'builtins' and not is_py3k:
+            # There are two implementations of `open` for either python 2/3.
+            # -> Rename the python2 version (`look at fake/builtins.pym`).
+            open_func = search_scope(module, 'open')
+            open_func.name = FakeName('open_python3')
+            open_func = search_scope(module, 'open_python2')
+            open_func.name = FakeName('open')
+        return module
+
+
+def search_scope(scope, obj_name):
+    for s in scope.subscopes:
+        if str(s.name) == obj_name:
+            return s
+
+
+def _faked(module, obj, name):
+    # Crazy underscore actions to try to escape all the internal madness.
+    if module is None:
+        try:
+            module = obj.__objclass__
+        except AttributeError:
+            pass
+
+        try:
+            imp_plz = obj.__module__
+        except AttributeError:
+            # Unfortunately in some cases like `int` there's no __module__
+            module = builtins
+        else:
+            module = __import__(imp_plz)
+
+    faked_mod = _load_faked_module(module)
+    if faked_mod is None:
+        return
+
+    # Having the module as a `parser.representation.module`, we need to scan
+    # for methods.
+    if name is None:
+        if inspect.isbuiltin(obj):
+            return search_scope(faked_mod, obj.__name__)
+        elif not inspect.isclass(obj):
+            # object is a method or descriptor
+            cls = search_scope(faked_mod, obj.__objclass__.__name__)
+            if cls is None:
+                return
+            return search_scope(cls, obj.__name__)
+    else:
+        if obj == module:
+            return search_scope(faked_mod, name)
+        else:
+            cls = search_scope(faked_mod, obj.__name__)
+            if cls is None:
+                return
+            return search_scope(cls, name)
+
+
+def get_faked(module, obj, name=None):
+    obj = obj.__class__ if is_class_instance(obj) else obj
+    result = _faked(module, obj, name)
+    if not isinstance(result, Class) and result is not None:
+        # Set the docstr which was previously not set (faked modules don't
+        # contain it).
+        result.docstr = obj.__doc__ or ''
+        return result
+
+
+def is_class_instance(obj):
+    """Like inspect.* methods."""
+    return not (inspect.isclass(obj) or inspect.ismodule(obj)
+                or inspect.isbuiltin(obj) or inspect.ismethod(obj)
+                or inspect.ismethoddescriptor(obj))

jedi/evaluate/compiled/fake/_weakref.pym

@@ -1,7 +1,7 @@
 def proxy(object, callback=None):
     return object
 
-class ref():
+class weakref():
     def __init__(self, object, callback=None):
         self.__object = object
     def __call__(self):

jedi/evaluate/compiled/fake/builtins.pym

@@ -45,6 +45,15 @@ class xrange():
         return 1
 
 
+def open(file, mode='r', buffering=-1, encoding=None, errors=None, newline=None, closefd=True):
+    import io
+    return io.TextIOWrapper(file, mode, buffering, encoding, errors, newline, closefd)
+
+
+def open_python2(name, mode=None, buffering=None):
+    return file(name, mode, buffering)
+
+
 #--------------------------------------------------------
 # descriptors
 #--------------------------------------------------------

jedi/evaluate/compiled/fake/io.pym

@@ -1,3 +1,3 @@
 class TextIOWrapper():
     def __next__(self):
-        return ''
+        return 'hacked io return'

jedi/evaluate/finder.py

@@ -7,7 +7,7 @@ from jedi import common
 from jedi import settings
 from jedi.evaluate import representation as er
 from jedi.evaluate import dynamic
-from jedi.evaluate import builtin
+from jedi.evaluate import compiled
 from jedi.evaluate import docstrings
 from jedi.evaluate import iterable
 
@@ -71,9 +71,10 @@ class NameFinder(object):
         if not result and isinstance(self.scope, er.Instance):
             # __getattr__ / __getattribute__
             for r in self._check_getattr(self.scope):
-                new_name = copy.copy(r.name)
-                new_name.parent = r
-                result.append(new_name)
+                if not isinstance(r, compiled.PyObject):
+                    new_name = copy.copy(r.name)
+                    new_name.parent = r
+                    result.append(new_name)
 
         debug.dbg('sfn filter "%s" in (%s-%s): %s@%s'
                   % (self.name_str, self.scope, nscope, u(result), self.position))
@@ -82,9 +83,8 @@ class NameFinder(object):
     def _check_getattr(self, inst):
         """Checks for both __getattr__ and __getattribute__ methods"""
         result = []
-        module = builtin.Builtin.scope
         # str is important to lose the NamePart!
-        name = pr.String(module, "'%s'" % self.name_str, (0, 0), (0, 0), inst)
+        name = compiled.create(str(self.name_str))
         with common.ignored(KeyError):
             result = inst.execute_subscope_by_name('__getattr__', [name])
         if not result:
@@ -242,7 +242,8 @@ class NameFinder(object):
                 c = r.expression_list()[0]
                 if c in ('*', '**'):
                     t = 'tuple' if c == '*' else 'dict'
-                    res_new = evaluator.execute(evaluator.find_types(builtin.Builtin.scope, t)[0])
+                    typ = evaluator.find_types(compiled.builtin, t)[0]
+                    res_new = evaluator.execute(typ)
             if not r.assignment_details:
                 # this means that there are no default params,
                 # so just ignore it.

jedi/evaluate/helpers.py

@@ -156,3 +156,20 @@ def scan_statement_for_calls(stmt, search_name, assignment_details=False):
                 s_new = s_new.next
 
     return result
+
+
+class FakeSubModule():
+    line_offset = 0
+
+
+class FakeStatement(pr.Statement):
+    def __init__(self, expression_list):
+        p = 0, 0
+        super(FakeStatement, self).__init__(FakeSubModule, expression_list, p, p)
+        self._expression_list = expression_list
+
+
+class FakeName(pr.Name):
+    def __init__(self, name, parent=None):
+        p = 0, 0
+        super(FakeName, self).__init__(FakeSubModule, [(name, p)], p, p, parent)

jedi/evaluate/imports.py

@@ -25,6 +25,7 @@ from jedi.parser import representation as pr
 from jedi.evaluate import sys_path
 from jedi import settings
 from jedi.common import source_to_unicode
+from jedi.evaluate import compiled
 
 
 class ModuleNotFound(Exception):
@@ -381,8 +382,9 @@ def strip_imports(evaluator, scopes):
 @cache.cache_star_import
 def remove_star_imports(evaluator, scope, ignored_modules=()):
     """
-    Check a module for star imports:
-    >>> from module import *
+    Check a module for star imports::
+
+        from module import *
 
     and follow these modules.
     """
@@ -404,9 +406,7 @@ def load_module(path=None, source=None, name=None):
                 with open(path) as f:
                     source = f.read()
         else:
-            # TODO refactoring remove
-            from jedi.evaluate import builtin
-            return builtin.BuiltinModule(path, name).parser.module
+            return compiled.load_module(path, name)
         p = path or name
         p = fast.FastParser(common.source_to_unicode(source), p)
         cache.save_parser(path, name, p)
@@ -436,7 +436,7 @@ def get_modules_containing_name(mods, name):
                 return load_module(path, source)
 
     # skip non python modules
-    mods = set(m for m in mods if m.path is None or m.path.endswith('.py'))
+    mods = set(m for m in mods if not isinstance(m, compiled.PyObject))
     mod_paths = set()
     for m in mods:
         mod_paths.add(m.path)

jedi/evaluate/iterable.py

@@ -3,9 +3,9 @@ import itertools
 from jedi import common
 from jedi import debug
 from jedi import settings
-from jedi._compatibility import use_metaclass, is_py3k
+from jedi._compatibility import use_metaclass, is_py3k, unicode
 from jedi.parser import representation as pr
-from jedi.evaluate import builtin
+from jedi.evaluate import compiled
 from jedi.evaluate import helpers
 from jedi.evaluate.cache import CachedMetaClass, memoize_default
 
@@ -24,16 +24,10 @@ class Generator(use_metaclass(CachedMetaClass, pr.Base)):
         content of a generator.
         """
         names = []
-        none_pos = (0, 0)
         executes_generator = ('__next__', 'send')
         for n in ('close', 'throw') + executes_generator:
-            name = pr.Name(builtin.Builtin.scope, [(n, none_pos)],
-                           none_pos, none_pos)
-            if n in executes_generator:
-                name.parent = self
-            else:
-                name.parent = builtin.Builtin.scope
-            names.append(name)
+            parent = self if n in executes_generator else compiled.builtin
+            names.append(helpers.FakeName(n, parent))
         debug.dbg('generator names', names)
         return names
 
@@ -78,16 +72,10 @@ class Array(use_metaclass(CachedMetaClass, pr.Base)):
                 # This is indexing only one element, with a fixed index number,
                 # otherwise it just ignores the index (e.g. [1+1]).
                 index = index_possibilities[0]
-
-                from jedi.evaluate.representation import Instance
-                if isinstance(index, Instance) \
-                        and str(index.name) in ['int', 'str'] \
-                        and len(index.var_args) == 1:
-                    # TODO this is just very hackish and a lot of use cases are
-                    # being ignored
-                    with common.ignored(KeyError, IndexError,
-                                        UnboundLocalError, TypeError):
-                        return self.get_exact_index_types(index.var_args[0])
+                if isinstance(index, compiled.PyObject) \
+                        and isinstance(index.obj, (int, str, unicode)):
+                    with common.ignored(KeyError, IndexError, TypeError):
+                        return self.get_exact_index_types(index.obj)
 
         result = list(self._follow_values(self._array.values))
         result += check_array_additions(self._evaluator, self)
@@ -108,6 +96,8 @@ class Array(use_metaclass(CachedMetaClass, pr.Base)):
                     str_key = key.value
                 elif isinstance(key, pr.Name):
                     str_key = str(key)
+                else:
+                    continue
 
                 if mixed_index == str_key:
                     index = i
@@ -130,17 +120,17 @@ class Array(use_metaclass(CachedMetaClass, pr.Base)):
         It returns e.g. for a list: append, pop, ...
         """
         # `array.type` is a string with the type, e.g. 'list'.
-        scope = self._evaluator.find_types(builtin.Builtin.scope, self._array.type)[0]
+        scope = self._evaluator.find_types(compiled.builtin, self._array.type)[0]
         scope = self._evaluator.execute(scope)[0]  # builtins only have one class
         names = scope.get_defined_names()
         return [ArrayMethod(n) for n in names]
 
     @property
     def parent(self):
-        return builtin.Builtin.scope
+        return compiled.builtin
 
     def get_parent_until(self):
-        return builtin.Builtin.scope
+        return compiled.builtin
 
     def __getattr__(self, name):
         if name not in ['type', 'start_pos', 'get_only_subelement', 'parent',
@@ -177,7 +167,7 @@ class ArrayMethod(object):
         return getattr(self.name, name)
 
     def get_parent_until(self):
-        return builtin.Builtin.scope
+        return compiled.builtin
 
     def __repr__(self):
         return "<%s of %s>" % (type(self).__name__, self.name)
@@ -240,7 +230,7 @@ def _check_array_additions(evaluator, compare_array, module, is_list):
     >>> a = [""]
     >>> a.append(1)
     """
-    if not settings.dynamic_array_additions or module.is_builtin():
+    if not settings.dynamic_array_additions or isinstance(module, compiled.PyObject):
         return []
 
     def check_calls(calls, add_name):

jedi/evaluate/param.py

@@ -2,8 +2,8 @@ import copy
 
 from jedi.parser import representation as pr
 from jedi.evaluate import iterable
-from jedi.evaluate import builtin
 from jedi.evaluate import common
+from jedi.evaluate import helpers
 
 
 def get_params(evaluator, func, var_args):
@@ -24,11 +24,11 @@ def get_params(evaluator, func, var_args):
             new_param.parent = parent
 
         # create an Array (-> needed for *args/**kwargs tuples/dicts)
-        arr = pr.Array(_FakeSubModule, start_pos, array_type, parent)
+        arr = pr.Array(helpers.FakeSubModule, start_pos, array_type, parent)
         arr.values = values
         key_stmts = []
         for key in keys:
-            stmt = pr.Statement(_FakeSubModule, [], start_pos, None)
+            stmt = pr.Statement(helpers.FakeSubModule, [], start_pos, None)
             stmt._expression_list = [key]
             key_stmts.append(stmt)
         arr.keys = key_stmts
@@ -140,8 +140,7 @@ def _var_args_iterator(evaluator, var_args):
                 continue
             old = stmt
             # generate a statement if it's not already one.
-            module = builtin.Builtin.scope
-            stmt = pr.Statement(module, [], (0, 0), None)
+            stmt = pr.Statement(helpers.FakeSubModule, [], (0, 0), None)
             stmt._expression_list = [old]
 
         # *args
@@ -156,7 +155,7 @@ def _var_args_iterator(evaluator, var_args):
                         yield None, field_stmt
                 elif isinstance(array, iterable.Generator):
                     for field_stmt in array.iter_content():
-                        yield None, _FakeStatement(field_stmt)
+                        yield None, helpers.FakeStatement([field_stmt])
         # **kwargs
         elif expression_list[0] == '**':
             for array in evaluator.eval_expression_list(expression_list[1:]):
@@ -177,13 +176,3 @@ def _var_args_iterator(evaluator, var_args):
                     yield key_arr[0].name, stmt
             else:
                 yield None, stmt
-
-
-class _FakeSubModule():
-    line_offset = 0
-
-
-class _FakeStatement(pr.Statement):
-    def __init__(self, content):
-        p = 0, 0
-        super(_FakeStatement, self).__init__(_FakeSubModule, [content], p, p)

jedi/evaluate/recursion.py

@@ -10,7 +10,7 @@ calls.
 from jedi.parser import representation as pr
 from jedi import debug
 from jedi import settings
-from jedi.evaluate import builtin
+from jedi.evaluate import compiled
 from jedi.evaluate import iterable
 
 
@@ -82,7 +82,7 @@ class _RecursionNode(object):
         # The same's true for the builtins, because the builtins are really
         # simple.
         self.is_ignored = isinstance(stmt, pr.Param) \
-            or (self.script == builtin.Builtin.scope)
+            or (self.script == compiled.builtin)
 
     def __eq__(self, other):
         if not other:
@@ -148,7 +148,7 @@ class ExecutionRecursionDetector(object):
         if isinstance(execution.base, (iterable.Array, iterable.Generator)):
             return False
         module = execution.get_parent_until()
-        if evaluate_generator or module == builtin.Builtin.scope:
+        if evaluate_generator or module == compiled.builtin:
             return False
 
         if in_par_execution_funcs:

jedi/evaluate/representation.py

@@ -16,7 +16,7 @@ from jedi.parser import representation as pr
 from jedi import debug
 from jedi import common
 from jedi.evaluate.cache import memoize_default, CachedMetaClass
-from jedi.evaluate import builtin
+from jedi.evaluate import compiled
 from jedi.evaluate import recursion
 from jedi.evaluate import iterable
 from jedi.evaluate import docstrings
@@ -49,7 +49,7 @@ class Instance(use_metaclass(CachedMetaClass, Executable)):
     def __init__(self, evaluator, base, var_args=()):
         super(Instance, self).__init__(evaluator, base, var_args)
         if str(base.name) in ['list', 'set'] \
-                and builtin.Builtin.scope == base.get_parent_until():
+                and compiled.builtin == base.get_parent_until():
             # compare the module path with the builtin name.
             self.var_args = iterable.check_array_instances(evaluator, self)
         else:
@@ -113,9 +113,10 @@ class Instance(use_metaclass(CachedMetaClass, Executable)):
                 if n.names[0] == self_name and len(n.names) == 2:
                     add_self_dot_name(n)
 
-        for s in self.base.get_super_classes():
-            for inst in self._evaluator.execute(s):
-                names += inst.get_self_attributes()
+        if not isinstance(self.base, compiled.PyObject):
+            for s in self.base.get_super_classes():
+                for inst in self._evaluator.execute(s):
+                    names += inst.get_self_attributes()
         return names
 
     def get_subscope_by_name(self, name):
@@ -141,8 +142,7 @@ class Instance(use_metaclass(CachedMetaClass, Executable)):
         """
         names = self.get_self_attributes()
 
-        class_names = self.base.instance_names()
-        for var in class_names:
+        for var in self.base.instance_names():
             names.append(InstanceElement(self._evaluator, self, var, True))
         return names
 
@@ -154,8 +154,7 @@ class Instance(use_metaclass(CachedMetaClass, Executable)):
         yield self, self.get_self_attributes()
 
         names = []
-        class_names = self.base.instance_names()
-        for var in class_names:
+        for var in self.base.instance_names():
             names.append(InstanceElement(self._evaluator, self, var, True))
         yield self, names
 
@@ -199,10 +198,10 @@ class InstanceElement(use_metaclass(CachedMetaClass, pr.Base)):
     def parent(self):
         par = self.var.parent
         if isinstance(par, Class) and par == self.instance.base \
-            or isinstance(par, pr.Class) \
+                or isinstance(par, pr.Class) \
                 and par == self.instance.base.base:
             par = self.instance
-        elif not isinstance(par, pr.Module):
+        elif not isinstance(par, (pr.Module, compiled.PyObject)):
             par = InstanceElement(self.instance._evaluator, self.instance, par, self.is_class_var)
         return par
 
@@ -256,13 +255,14 @@ class Class(use_metaclass(CachedMetaClass, pr.IsScope)):
                     debug.warning('Received non class, as a super class')
                     continue  # Just ignore other stuff (user input error).
                 supers.append(cls)
-        if not supers and self.base.parent != builtin.Builtin.scope:
+        if not supers and self.base.parent != compiled.builtin:
             # add `object` to classes
-            supers += self._evaluator.find_types(builtin.Builtin.scope, 'object')
+            supers += self._evaluator.find_types(compiled.builtin, 'object')
         return supers
 
     @memoize_default(default=())
     def instance_names(self):
+        # TODO REMOVE instance_names
         def in_iterable(name, iterable):
             """ checks if the name is in the variable 'iterable'. """
             for i in iterable:
@@ -277,17 +277,20 @@ class Class(use_metaclass(CachedMetaClass, pr.IsScope)):
         # TODO mro!
         for cls in self.get_super_classes():
             # Get the inherited names.
-            for i in cls.instance_names():
-                if not in_iterable(i, result):
-                    super_result.append(i)
+            if isinstance(cls, compiled.PyObject):
+                super_result += cls.get_defined_names()
+            else:
+                for i in cls.instance_names():
+                    if not in_iterable(i, result):
+                        super_result.append(i)
         result += super_result
         return result
 
     @memoize_default(default=())
     def get_defined_names(self):
         result = self.instance_names()
-        type_cls = self._evaluator.find_types(builtin.Builtin.scope, 'type')[0]
-        return result + type_cls.base.get_defined_names()
+        type_cls = self._evaluator.find_types(compiled.builtin, 'type')[0]
+        return result + list(type_cls.get_defined_names())
 
     def get_subscope_by_name(self, name):
         for sub in reversed(self.subscopes):
@@ -373,11 +376,11 @@ class Function(use_metaclass(CachedMetaClass, pr.IsScope)):
             return Function(self._evaluator, self.base_func, True)
         return decorated_func
 
-    def get_magic_method_names(self):
-        return builtin.Builtin.magic_function_scope(self._evaluator).get_defined_names()
+    def get_magic_function_names(self):
+        return compiled.magic_function_class.get_defined_names()
 
-    def get_magic_method_scope(self):
-        return builtin.Builtin.magic_function_scope(self._evaluator)
+    def get_magic_function_scope(self):
+        return compiled.magic_function_class
 
     def __getattr__(self, name):
         return getattr(self.base_func, name)

jedi/evaluate/stdlib.py

@@ -3,7 +3,8 @@ Implementations of standard library functions, because it's not possible to
 understand them with Jedi.
 """
 
-from jedi.evaluate import builtin
+from jedi._compatibility import unicode
+from jedi.evaluate import compiled
 from jedi.evaluate import representation as er
 from jedi.evaluate import iterable
 from jedi.parser import representation as pr
@@ -17,7 +18,7 @@ class NotInStdLib(LookupError):
 def execute(evaluator, obj, params):
     if not isinstance(obj, (iterable.Generator, iterable.Array)):
         obj_name = str(obj.name)
-        if obj.parent == builtin.Builtin.scope:
+        if obj.parent == compiled.builtin:
             # for now we just support builtin functions.
             try:
                 return _implemented['builtins'][obj_name](evaluator, obj, params)
@@ -44,18 +45,17 @@ def builtins_getattr(evaluator, obj, params):
     objects = _follow_param(evaluator, params, 0)
     names = _follow_param(evaluator, params, 1)
     for obj in objects:
-        if not isinstance(obj, (er.Instance, er.Class, pr.Module)):
+        if not isinstance(obj, (er.Instance, er.Class, pr.Module, compiled.PyObject)):
             debug.warning('getattr called without instance')
             continue
 
-        for arr_name in names:
-            if not isinstance(arr_name, er.Instance):
+        for name in names:
+            s = unicode, str
+            if isinstance(name, compiled.PyObject) and isinstance(name.obj, s):
+                stmts += evaluator.follow_path(iter([name.obj]), [obj], obj)
+            else:
                 debug.warning('getattr called without str')
                 continue
-            if len(arr_name.var_args) != 1:
-                debug.warning('jedi getattr is too simple')
-            key = arr_name.var_args[0]
-            stmts += evaluator.follow_path(iter([key]), [obj], obj)
     return stmts
 
 

jedi/keywords.py

@@ -2,9 +2,8 @@ import pydoc
 import keyword
 
 from jedi._compatibility import is_py3k
-from jedi.parser import representation as pr
 from jedi import common
-from jedi.evaluate import builtin
+from jedi.evaluate import compiled
 
 try:
     from pydoc_data import topics as pydoc_topics
@@ -30,8 +29,7 @@ def keyword_names(*args, **kwargs):
     kwds = []
     for k in keywords(*args, **kwargs):
         start = k.start_pos
-        end = start[0], start[1] + len(k.name)
-        kwds.append(pr.Name(k.parent, [(k.name, start)], start, end, k))
+        kwds.append(KeywordName(k, k.name, start))
     return kwds
 
 
@@ -39,11 +37,22 @@ def get_operator(string, pos):
     return Keyword(string, pos)
 
 
+class KeywordName(object):
+    def __init__(self, parent, name, start_pos):
+        self.parent = parent
+        self.names = [name]
+        self.start_pos = start_pos
+
+    @property
+    def end_pos(self):
+        return self.start_pos[0], self.start_pos[1] + len(self.name)
+
+
 class Keyword(object):
     def __init__(self, name, pos):
         self.name = name
         self.start_pos = pos
-        self.parent = builtin.Builtin.scope
+        self.parent = compiled.builtin
 
     def get_parent_until(self):
         return self.parent

jedi/parser/representation.py

@@ -362,17 +362,13 @@ class SubModule(Scope, Module):
             string = ''  # no path -> empty name
         else:
             sep = (re.escape(os.path.sep),) * 2
-            r = re.search(r'([^%s]*?)(%s__init__)?(\.py|\.so)?$' % sep,
-                          self.path)
+            r = re.search(r'([^%s]*?)(%s__init__)?(\.py|\.so)?$' % sep, self.path)
             # remove PEP 3149 names
             string = re.sub('\.[a-z]+-\d{2}[mud]{0,3}$', '', r.group(1))
         # positions are not real therefore choose (0, 0)
         names = [(string, (0, 0))]
         return Name(self, names, (0, 0), (0, 0), self.use_as_parent)
 
-    def is_builtin(self):
-        return not (self.path is None or self.path.endswith('.py'))
-
     @property
     def has_explicit_absolute_import(self):
         """
@@ -1239,9 +1235,6 @@ class Literal(StatementElement):
     def get_code(self):
         return self.literal + super(Literal, self).get_code()
 
-    def type_as_string(self):
-        return type(self.value).__name__
-
     def __repr__(self):
         if is_py3k:
             s = self.literal

pytest.ini

@@ -2,7 +2,7 @@
 addopts = --doctest-modules
 
 # Ignore broken files in blackbox test directories
-norecursedirs = .* docs completion refactor absolute_import namespace_package
+norecursedirs = .* docs completion refactor absolute_import namespace_package scripts
 
 # Activate `clean_jedi_cache` fixture for all tests.  This should be
 # fine as long as we are using `clean_jedi_cache` as a session scoped

scripts/memory_check.py

@@ -0,0 +1,53 @@
+"""
+This is a convenience script to test the speed and memory usage of Jedi with
+large libraries.
+
+Each library is preloaded by jedi, recording the time and memory consumed by
+each operation.
+
+You can provide additional libraries via command line arguments.
+
+Note: This requires the psutil library, available on PyPI.
+"""
+import time
+import sys
+import psutil
+import jedi
+
+
+def used_memory():
+    """Return the total MB of System Memory in use."""
+    return psutil.virtual_memory().used / 2**20
+
+
+def profile_preload(mod):
+    """Preload a module into Jedi, recording time and memory used."""
+    base = used_memory()
+    t0 = time.time()
+    jedi.preload_module(mod)
+    elapsed = time.time() - t0
+    used = used_memory() - base
+    return elapsed, used
+
+
+def main(mods):
+    """Preload the modules, and print the time and memory used."""
+    t0 = time.time()
+    baseline = used_memory()
+    print('Time (s) | Mem (MB) | Package')
+    print('------------------------------')
+    for mod in mods:
+        elapsed, used = profile_preload(mod)
+        if used > 0:
+            print('%8.1f | %8d | %s' % (elapsed, used, mod))
+    print('------------------------------')
+    elapsed = time.time() - t0
+    used = used_memory() - baseline
+    print('%8.1f | %8d | %s' % (elapsed, used, 'Total'))
+
+
+if __name__ == '__main__':
+    mods = ['re', 'numpy', 'scipy', 'scipy.sparse', 'scipy.stats',
+            'wx', 'decimal', 'PyQt4.QtGui', 'PySide.QtGui', 'Tkinter']
+    mods += sys.argv[1:]
+    main(mods)

setup.py

@@ -26,8 +26,8 @@ setup(name='jedi',
       license='MIT',
       keywords='python completion refactoring vim',
       long_description=readme,
-      packages=['jedi', 'jedi.parser', 'jedi.evaluate', 'jedi.api'],
-      package_data={'jedi': ['evlaluate/evaluate/mixin/*.pym']},
+      packages=['jedi', 'jedi.parser', 'jedi.evaluate', 'jedi.evaluate.compiled', 'jedi.api'],
+      package_data={'jedi': ['evlaluate/evaluate/compiled/fake/*.pym']},
       platforms=['any'],
       classifiers=[
           'Development Status :: 4 - Beta',

test/test_api.py

@@ -2,27 +2,25 @@
 Test all things related to the ``jedi.api`` module.
 """
 
-from jedi import common, api
+from jedi import api
 from pytest import raises
 
 
 def test_preload_modules():
     def check_loaded(*modules):
-        # + 1 for builtin, +1 for None module (currently used)
-        assert len(new) == len(modules) + 2
-        for i in modules + ('__builtin__',):
-            assert [i in k for k in new.keys() if k is not None]
+        # +1 for None module (currently used)
+        assert len(parser_cache) == len(modules) + 1
+        for i in modules:
+            assert [i in k for k in parser_cache.keys() if k is not None]
 
     from jedi import cache
     temp_cache, cache.parser_cache = cache.parser_cache, {}
-    new = cache.parser_cache
-    with common.ignored(KeyError): # performance of tests -> no reload
-        new['__builtin__'] = temp_cache['__builtin__']
+    parser_cache = cache.parser_cache
 
-    api.preload_module('datetime')
-    check_loaded('datetime')
+    api.preload_module('sys')
+    check_loaded()  # compiled (c_builtin) modules shouldn't be in the cache.
     api.preload_module('json', 'token')
-    check_loaded('datetime', 'json', 'token')
+    check_loaded('json', 'token')
 
     cache.parser_cache = temp_cache
 
@@ -30,6 +28,7 @@ def test_preload_modules():
 def test_empty_script():
     assert api.Script('')
 
+
 def test_line_number_errors():
     """
     Script should raise a ValueError if line/column numbers are not in a

test/test_api_classes.py

@@ -15,6 +15,7 @@ def test_is_keyword():
     results = Script('str', 1, 1, None).goto_definitions()
     assert len(results) == 1 and results[0].is_keyword is False
 
+
 def make_definitions():
     """
     Return a list of definitions for parametrized tests.

test/test_builtin.py

@@ -1,13 +0,0 @@
-from jedi.evaluate import builtin
-
-
-def test_parse_function_doc_illegal_docstr():
-
-    def test_func(a):
-        """
-        test_func(o
-
-        doesn't have a closing bracket.
-        """
-
-    assert ('', '') == builtin._parse_function_doc(test_func)

test/test_call_signatures.py

@@ -15,7 +15,7 @@ class TestCallSignatures(TestCase):
             assert signatures[0].call_name == expected_name
             assert signatures[0].index == expected_index
 
-    def test_call_signatures(self):
+    def test_simple(self):
         def run(source, name, index=0, column=None, line=1):
             self._run(source, name, index, line, column)
 
@@ -72,7 +72,7 @@ class TestCallSignatures(TestCase):
             "func(alpha='101',"
         run(s, 'func', 0, column=13, line=2)
 
-    def test_function_definition_complex(self):
+    def test_complex(self):
         s = """
                 def abc(a,b):
                     pass
@@ -106,7 +106,7 @@ class TestCallSignatures(TestCase):
         # just don't throw an exception (if numpy doesn't exist, just ignore it)
         assert Script(s).call_signatures() == []
 
-    def test_function_definition_empty_paren_pre_space(self):
+    def test_call_signatures_empty_parentheses_pre_space(self):
         s = textwrap.dedent("""\
         def f(a, b):
             pass

test/test_compiled.py

@@ -0,0 +1,36 @@
+from jedi._compatibility import builtins
+from jedi.parser.representation import Function
+from jedi.evaluate import compiled
+from jedi.evaluate import Evaluator
+
+
+def test_simple():
+    e = Evaluator()
+    bltn = compiled.PyObject(builtins)
+    obj = compiled.PyObject('_str_', bltn)
+    upper = e.find_types(obj, 'upper')
+    assert len(upper) == 1
+    objs = list(e.execute(upper[0]))
+    assert len(objs) == 1
+    assert objs[0].obj is str
+
+
+def test_fake_loading():
+    assert isinstance(compiled.create(next), Function)
+
+    string = compiled.builtin.get_subscope_by_name('str')
+    from_name = compiled._create_from_name(compiled.builtin, string, '__init__')
+    assert isinstance(from_name, Function)
+
+
+def test_fake_docstr():
+    assert compiled.create(next).docstr == next.__doc__
+
+
+def test_parse_function_doc_illegal_docstr():
+    docstr = """
+    test_func(o
+
+    doesn't have a closing bracket.
+    """
+    assert ('', '') == compiled._parse_function_doc(docstr)