Merge branch 'dev' into get_code_fidelity

Conflicts:
	jedi/evaluate/docstrings.py
	jedi/parser/representation.py

Resolving merge problems:
* Introducing docstring for compiled.fake
* Partly fixing poor decision in TokenDocstring __init__

jedi/evaluate/compiled/__init__.py

@@ -0,0 +1,308 @@
+"""
+Imitate the parser representation.
+"""
+import inspect
+import re
+import sys
+import os
+
+from jedi._compatibility import builtins as _builtins
+from jedi import debug
+from jedi.parser.representation import Base
+from jedi.cache import underscore_memoization
+from jedi.evaluate.sys_path import get_sys_path
+from . import fake
+
+
+class CompiledObject(Base):
+    # comply with the parser
+    start_pos = 0, 0
+    asserts = []
+    path = None  # modules have this attribute - set it to None.
+
+    def __init__(self, obj, parent=None):
+        self.obj = obj
+        self.parent = parent
+        self.doc = inspect.getdoc(obj)
+
+    def __repr__(self):
+        return '<%s: %s>' % (type(self).__name__, self.obj)
+
+    def get_parent_until(self, *args, **kwargs):
+        # compiled modules only use functions and classes/methods (2 levels)
+        return getattr(self.parent, 'parent', self.parent) or self.parent or self
+
+    @underscore_memoization
+    def _parse_function_doc(self):
+        if self.doc is None:
+            return '', ''
+
+        return _parse_function_doc(self.doc)
+
+    def type(self):
+        cls = self._cls().obj
+        if inspect.isclass(cls):
+            return 'class'
+        elif inspect.ismodule(cls):
+            return 'module'
+        elif inspect.isbuiltin(cls) or inspect.ismethod(cls) \
+                or inspect.ismethoddescriptor(cls):
+            return 'def'
+
+    def is_executable_class(self):
+        return inspect.isclass(self.obj)
+
+    @underscore_memoization
+    def _cls(self):
+        # Ensures that a CompiledObject is returned that is not an instance (like list)
+        if fake.is_class_instance(self.obj):
+            try:
+                c = self.obj.__class__
+            except AttributeError:
+                # happens with numpy.core.umath._UFUNC_API (you get it
+                # automatically by doing `import numpy`.
+                c = type(None)
+            return CompiledObject(c, self.parent)
+        return self
+
+    def get_defined_names(self):
+        cls = self._cls()
+        for name in dir(cls.obj):
+            yield CompiledName(cls, name)
+
+    def instance_names(self):
+        return self.get_defined_names()
+
+    def get_subscope_by_name(self, name):
+        if name in dir(self._cls().obj):
+            return CompiledName(self._cls(), name).parent
+        else:
+            raise KeyError("CompiledObject doesn't have an attribute '%s'." % name)
+
+    @property
+    def name(self):
+        # might not exist sometimes (raises AttributeError)
+        return self._cls().obj.__name__
+
+    def execute_function(self, evaluator, params):
+        if self.type() != 'def':
+            return
+
+        for name in self._parse_function_doc()[1].split():
+            try:
+                bltn_obj = _create_from_name(builtin, builtin, name)
+            except AttributeError:
+                continue
+            else:
+                if isinstance(bltn_obj, CompiledObject):
+                    # We want everything except None.
+                    if bltn_obj.obj is not None:
+                        yield bltn_obj
+                else:
+                    for result in evaluator.execute(bltn_obj, params):
+                        yield result
+
+    @property
+    @underscore_memoization
+    def subscopes(self):
+        """
+        Returns only the faked scopes - the other ones are not important for
+        internal analysis.
+        """
+        module = self.get_parent_until()
+        faked_subscopes = []
+        for name in dir(self._cls().obj):
+            f = fake.get_faked(module.obj, self.obj, name)
+            if f:
+                f.parent = self
+                faked_subscopes.append(f)
+        return faked_subscopes
+
+    def get_self_attributes(self):
+        return []  # Instance compatibility
+
+    def get_imports(self):
+        return []  # Builtins don't have imports
+
+
+class CompiledName(object):
+    def __init__(self, obj, name):
+        self._obj = obj
+        self.name = name
+        self.start_pos = 0, 0  # an illegal start_pos, to make sorting easy.
+
+    def get_parent_until(self):
+        return self.parent.get_parent_until()
+
+    def __str__(self):
+        return self.name
+
+    def __repr__(self):
+        return '<%s: (%s).%s>' % (type(self).__name__, self._obj.name, self.name)
+
+    @property
+    @underscore_memoization
+    def parent(self):
+        module = self._obj.get_parent_until()
+        return _create_from_name(module, self._obj, self.name)
+
+    @property
+    def names(self):
+        return [self.name]  # compatibility with parser.representation.Name
+
+    def get_code(self):
+        return self.name
+
+
+def load_module(path, name):
+    if not name:
+        name = os.path.basename(path)
+        name = name.rpartition('.')[0]  # cut file type (normally .so)
+
+    # sometimes there are endings like `_sqlite3.cpython-32mu`
+    name = re.sub(r'\..*', '', name)
+
+    dot_path = []
+    if path:
+        p = path
+        # if path is not in sys.path, we need to make a well defined import
+        # like `from numpy.core import umath.`
+        while p and p not in sys.path:
+            p, sep, mod = p.rpartition(os.path.sep)
+            dot_path.insert(0, mod.partition('.')[0])
+        if p:
+            name = ".".join(dot_path)
+            path = p
+        else:
+            path = os.path.dirname(path)
+
+    sys_path = get_sys_path()
+    if path:
+        sys_path.insert(0, path)
+
+    temp, sys.path = sys.path, sys_path
+    try:
+        module = __import__(name, {}, {}, dot_path[:-1])
+    except AttributeError:
+        # use sys.modules, because you cannot access some modules
+        # directly. -> github issue #59
+        module = sys.modules[name]
+    sys.path = temp
+    return CompiledObject(module)
+
+
+docstr_defaults = {
+    'floating point number': 'float',
+    'character': 'str',
+    'integer': 'int',
+    'dictionary': 'dict',
+    'string': 'str',
+}
+
+
+def _parse_function_doc(doc):
+    """
+    Takes a function and returns the params and return value as a tuple.
+    This is nothing more than a docstring parser.
+
+    TODO docstrings like utime(path, (atime, mtime)) and a(b [, b]) -> None
+    TODO docstrings like 'tuple of integers'
+    """
+    # parse round parentheses: def func(a, (b,c))
+    try:
+        count = 0
+        start = doc.index('(')
+        for i, s in enumerate(doc[start:]):
+            if s == '(':
+                count += 1
+            elif s == ')':
+                count -= 1
+            if count == 0:
+                end = start + i
+                break
+        param_str = doc[start + 1:end]
+    except (ValueError, UnboundLocalError):
+        # ValueError for doc.index
+        # UnboundLocalError for undefined end in last line
+        debug.dbg('no brackets found - no param')
+        end = 0
+        param_str = ''
+    else:
+        # remove square brackets, that show an optional param ( = None)
+        def change_options(m):
+            args = m.group(1).split(',')
+            for i, a in enumerate(args):
+                if a and '=' not in a:
+                    args[i] += '=None'
+            return ','.join(args)
+
+        while True:
+            param_str, changes = re.subn(r' ?\[([^\[\]]+)\]',
+                                         change_options, param_str)
+            if changes == 0:
+                break
+    param_str = param_str.replace('-', '_')  # see: isinstance.__doc__
+
+    # parse return value
+    r = re.search('-[>-]* ', doc[end:end + 7])
+    if r is None:
+        ret = ''
+    else:
+        index = end + r.end()
+        # get result type, which can contain newlines
+        pattern = re.compile(r'(,\n|[^\n-])+')
+        ret_str = pattern.match(doc, index).group(0).strip()
+        # New object -> object()
+        ret_str = re.sub(r'[nN]ew (.*)', r'\1()', ret_str)
+
+        ret = docstr_defaults.get(ret_str, ret_str)
+
+    return param_str, ret
+
+
+class Builtin(CompiledObject):
+    def get_defined_names(self):
+        # Filter None, because it's really just a keyword, nobody wants to
+        # access it.
+        return [d for d in super(Builtin, self).get_defined_names() if d.name != 'None']
+
+
+def _a_generator(foo):
+    """Used to have an object to return for generators."""
+    yield 42
+    yield foo
+
+builtin = Builtin(_builtins)
+magic_function_class = CompiledObject(type(load_module), parent=builtin)
+generator_obj = CompiledObject(_a_generator(1.0))
+
+
+def _create_from_name(module, parent, name):
+    faked = fake.get_faked(module.obj, parent.obj, name)
+    # only functions are necessary.
+    if faked is not None:
+        faked.parent = parent
+        return faked
+
+    try:
+        obj = getattr(parent.obj, name)
+    except AttributeError:
+        # happens e.g. in properties of
+        # PyQt4.QtGui.QStyleOptionComboBox.currentText
+        # -> just set it to None
+        obj = None
+    return CompiledObject(obj, parent)
+
+
+def create(obj, parent=builtin, module=None):
+    """
+    A very weird interface class to this module. The more options provided the
+    more acurate loading compiled objects is.
+    """
+    if not inspect.ismodule(obj):
+        faked = fake.get_faked(module and module.obj, obj)
+        if faked is not None:
+            faked.parent = parent
+            return faked
+
+    return CompiledObject(obj, parent)

jedi/evaluate/compiled/fake.py

@@ -0,0 +1,117 @@
+"""
+Loads functions that are mixed in to the standard library. E.g. builtins are
+written in C (binaries), but my autocompletion only understands Python code. By
+mixing in Python code, the autocompletion should work much better for builtins.
+"""
+
+import os
+import inspect
+
+from jedi._compatibility import is_py3, builtins
+from jedi.parser import Parser
+from jedi.parser import token as token_pr
+from jedi.parser.representation import Class
+from jedi.evaluate.helpers import FakeName
+
+modules = {}
+
+
+def _load_faked_module(module):
+    module_name = module.__name__
+    if module_name == '__builtin__' and not is_py3:
+        module_name = 'builtins'
+
+    try:
+        return modules[module_name]
+    except KeyError:
+        path = os.path.dirname(os.path.abspath(__file__))
+        try:
+            with open(os.path.join(path, 'fake', module_name) + '.pym') as f:
+                source = f.read()
+        except IOError:
+            modules[module_name] = None
+            return
+        module = Parser(source, module_name).module
+        modules[module_name] = module
+
+        if module_name == 'builtins' and not is_py3:
+            # There are two implementations of `open` for either python 2/3.
+            # -> Rename the python2 version (`look at fake/builtins.pym`).
+            open_func = search_scope(module, 'open')
+            open_func.name = FakeName('open_python3')
+            open_func = search_scope(module, 'open_python2')
+            open_func.name = FakeName('open')
+        return module
+
+
+def search_scope(scope, obj_name):
+    for s in scope.subscopes:
+        if str(s.name) == obj_name:
+            return s
+
+
+def get_module(obj):
+    if inspect.ismodule(obj):
+        return obj
+    try:
+        obj = obj.__objclass__
+    except AttributeError:
+        pass
+
+    try:
+        imp_plz = obj.__module__
+    except AttributeError:
+        # Unfortunately in some cases like `int` there's no __module__
+        return builtins
+    else:
+        return __import__(imp_plz)
+
+
+def _faked(module, obj, name):
+    # Crazy underscore actions to try to escape all the internal madness.
+    if module is None:
+        module = get_module(obj)
+
+    faked_mod = _load_faked_module(module)
+    if faked_mod is None:
+        return
+
+    # Having the module as a `parser.representation.module`, we need to scan
+    # for methods.
+    if name is None:
+        if inspect.isbuiltin(obj):
+            return search_scope(faked_mod, obj.__name__)
+        elif not inspect.isclass(obj):
+            # object is a method or descriptor
+            cls = search_scope(faked_mod, obj.__objclass__.__name__)
+            if cls is None:
+                return
+            return search_scope(cls, obj.__name__)
+    else:
+        if obj == module:
+            return search_scope(faked_mod, name)
+        else:
+            cls = search_scope(faked_mod, obj.__name__)
+            if cls is None:
+                return
+            return search_scope(cls, name)
+
+
+def get_faked(module, obj, name=None):
+    obj = obj.__class__ if is_class_instance(obj) else obj
+    result = _faked(module, obj, name)
+    if not isinstance(result, Class) and result is not None:
+        # Set the docstr which was previously not set (faked modules don't
+        # contain it).
+        result.docstr = None
+        if obj.__doc__:
+            result.docstr = token_pr.TokenDocstring.fake_docstring(obj.__doc__)
+        return result
+
+
+def is_class_instance(obj):
+    """Like inspect.* methods."""
+    return not (inspect.isclass(obj) or inspect.ismodule(obj)
+                or inspect.isbuiltin(obj) or inspect.ismethod(obj)
+                or inspect.ismethoddescriptor(obj) or inspect.iscode(obj)
+                or inspect.isgenerator(obj))

jedi/evaluate/compiled/fake/_functools.pym

@@ -0,0 +1,9 @@
+class partial():
+    def __init__(self, func, *args, **keywords):
+        self.__func = func
+        self.__args = args
+        self.__keywords = keywords
+
+    def __call__(self, *args, **kwargs):
+        # I know this doesn't work in Python, but Jedi can this ;-)
+        return self.__func(*self.__args, *args, **self.keywords, **kwargs)

jedi/evaluate/compiled/fake/_sqlite3.pym

@@ -0,0 +1,26 @@
+def connect(database, timeout=None, isolation_level=None, detect_types=None, factory=None):
+    return Connection()
+
+
+class Connection():
+    def cursor(self):
+        return Cursor()
+
+
+class Cursor():
+    def cursor(self):
+        return Cursor()
+
+    def fetchone(self):
+        return Row()
+
+    def fetchmany(self, size=cursor.arraysize):
+        return [self.fetchone()]
+
+    def fetchall(self):
+        return [self.fetchone()]
+
+
+class Row():
+    def keys(self):
+        return ['']

jedi/evaluate/compiled/fake/_sre.pym

@@ -0,0 +1,99 @@
+def compile():
+    class SRE_Match():
+        endpos = 1
+        lastgroup = 0
+        lastindex = 1
+        pos = 0
+        string = 'a'
+        regs = ((0, 1),)
+
+        def __init__(self, pattern):
+            self.re = pattern
+
+        def start(self):
+            return 0
+
+        def end(self):
+            return 1
+
+        def span(self):
+            return 0, 1
+
+        def expand(self):
+            return ''
+
+        def group(self, nr):
+            return ''
+
+        def groupdict(self):
+            return {'a', 'a'}
+
+        def groups(self):
+            return ('a',)
+
+    class SRE_Pattern():
+        flags = 0
+        groupindex = {}
+        groups = 0
+        pattern = 'a'
+
+        def findall(self, string, pos=None, endpos=None):
+            """
+            findall(string[, pos[, endpos]]) --> list.
+            Return a list of all non-overlapping matches of pattern in string.
+            """
+            return ['a']
+
+        def finditer(self, string, pos=None, endpos=None):
+            """
+            finditer(string[, pos[, endpos]]) --> iterator.
+            Return an iterator over all non-overlapping matches for the
+            RE pattern in string. For each match, the iterator returns a
+            match object.
+            """
+            yield SRE_Match(self)
+
+        def match(self, string, pos=None, endpos=None):
+            """
+            match(string[, pos[, endpos]]) --> match object or None.
+            Matches zero or more characters at the beginning of the string
+            pattern
+            """
+            return SRE_Match(self)
+
+        def scanner(self, string, pos=None, endpos=None):
+            pass
+
+        def search(self, string, pos=None, endpos=None):
+            """
+            search(string[, pos[, endpos]]) --> match object or None.
+            Scan through string looking for a match, and return a corresponding
+            MatchObject instance. Return None if no position in the string matches.
+            """
+            return SRE_Match(self)
+
+        def split(self, string, maxsplit=0]):
+            """
+            split(string[, maxsplit = 0])  --> list.
+            Split string by the occurrences of pattern.
+            """
+            return ['a']
+
+        def sub(self, repl, string, count=0):
+            """
+            sub(repl, string[, count = 0]) --> newstring
+            Return the string obtained by replacing the leftmost non-overlapping
+            occurrences of pattern in string by the replacement repl.
+            """
+            return ''
+
+        def subn(self, repl, string, count=0):
+            """
+            subn(repl, string[, count = 0]) --> (newstring, number of subs)
+            Return the tuple (new_string, number_of_subs_made) found by replacing
+            the leftmost non-overlapping occurrences of pattern with the
+            replacement repl.
+            """
+            return ('', 1)
+
+    return SRE_Pattern()

jedi/evaluate/compiled/fake/_weakref.pym

@@ -0,0 +1,8 @@
+def proxy(object, callback=None):
+    return object
+
+class weakref():
+    def __init__(self, object, callback=None):
+        self.__object = object
+    def __call__(self):
+        return self.__object

jedi/evaluate/compiled/fake/builtins.pym

@@ -0,0 +1,237 @@
+"""
+Pure Python implementation of some builtins.
+This code is not going to be executed anywhere.
+These implementations are not always correct, but should work as good as
+possible for the auto completion.
+"""
+
+
+def next(iterator, default=None):
+    if hasattr("next"):
+        return iterator.next()
+    else:
+        return iterator.__next__()
+    return default
+
+
+def iter(collection, sentinel=None):
+    if sentinel:
+        yield collection()
+    else:
+        for c in collection:
+            yield c
+
+
+def range(start, stop=None, step=1):
+    return [0]
+
+
+class file():
+    def __iter__(self):
+        yield ''
+    def next(self):
+        return ''
+
+
+class xrange():
+    # Attention: this function doesn't exist in Py3k (there it is range).
+    def __iter__(self):
+        yield 1
+
+    def count(self):
+        return 1
+
+    def index(self):
+        return 1
+
+
+def open(file, mode='r', buffering=-1, encoding=None, errors=None, newline=None, closefd=True):
+    import io
+    return io.TextIOWrapper(file, mode, buffering, encoding, errors, newline, closefd)
+
+
+def open_python2(name, mode=None, buffering=None):
+    return file(name, mode, buffering)
+
+
+#--------------------------------------------------------
+# descriptors
+#--------------------------------------------------------
+class property():
+    def __init__(self, fget, fset=None, fdel=None, doc=None):
+        self.fget = fget
+        self.fset = fset
+        self.fdel = fdel
+        self.__doc__ = doc
+
+    def __get__(self, obj, cls):
+        return self.fget(obj)
+
+    def __set__(self, obj, value):
+        self.fset(obj, value)
+
+    def __delete__(self, obj):
+        self.fdel(obj)
+
+    def setter(self, func):
+        self.fset = func
+        return self
+
+    def getter(self, func):
+        self.fget = func
+        return self
+
+    def deleter(self, func):
+        self.fdel = func
+        return self
+
+
+class staticmethod():
+    def __init__(self, func):
+        self.__func = func
+
+    def __get__(self, obj, cls):
+        return self.__func
+
+
+class classmethod():
+    def __init__(self, func):
+        self.__func = func
+
+    def __get__(self, obj, cls):
+        def _method(*args, **kwargs):
+            return self.__func(cls, *args, **kwargs)
+        return _method
+
+
+#--------------------------------------------------------
+# array stuff
+#--------------------------------------------------------
+class list():
+    def __init__(self, iterable=[]):
+        self.__iterable = []
+        for i in iterable:
+            self.__iterable += [i]
+
+    def __iter__(self):
+        for i in self.__iterable:
+            yield i
+
+    def __getitem__(self, y):
+        return self.__iterable[y]
+
+    def pop(self):
+        return self.__iterable[-1]
+
+
+class tuple():
+    def __init__(self, iterable=[]):
+        self.__iterable = []
+        for i in iterable:
+            self.__iterable += [i]
+
+    def __iter__(self):
+        for i in self.__iterable:
+            yield i
+
+    def __getitem__(self, y):
+        return self.__iterable[y]
+
+    def index(self):
+        return 1
+
+    def count(self):
+        return 1
+
+
+class set():
+    def __init__(self, iterable=[]):
+        self.__iterable = iterable
+
+    def __iter__(self):
+        for i in self.__iterable:
+            yield i
+
+    def pop(self):
+        return self.__iterable.pop()
+
+    def copy(self):
+        return self
+
+    def difference(self, other):
+        return self - other
+
+    def intersection(self, other):
+        return self & other
+
+    def symmetric_difference(self, other):
+        return self ^ other
+
+    def union(self, other):
+        return self | other
+
+
+class frozenset():
+    def __init__(self, iterable=[]):
+        self.__iterable = iterable
+
+    def __iter__(self):
+        for i in self.__iterable:
+            yield i
+
+    def copy(self):
+        return self
+
+
+class dict():
+    def __init__(self, **elements):
+        self.__elements = elements
+
+    def clear(self):
+        # has a strange docstr
+        pass
+
+    def get(self, k, d=None):
+        # TODO implement
+        try:
+            #return self.__elements[k]
+            pass
+        except KeyError:
+            return d
+
+
+class reversed():
+    def __init__(self, sequence):
+        self.__sequence = sequence
+
+    def __iter__(self):
+        for i in self.__sequence:
+            yield i
+
+    def __next__(self):
+        return next(self.__iter__())
+
+    def next(self):
+        return next(self.__iter__())
+
+
+def sorted(iterable, cmp=None, key=None, reverse=False):
+    return iterable
+
+
+#--------------------------------------------------------
+# basic types
+#--------------------------------------------------------
+class int():
+    def __init__(self, x, base=None):
+        pass
+
+
+class str():
+    def __init__(self, obj):
+        pass
+
+
+class type():
+    def mro():
+        return [object]

jedi/evaluate/compiled/fake/datetime.pym

@@ -0,0 +1,4 @@
+class datetime():
+    @staticmethod
+    def now():
+        return datetime()

jedi/evaluate/compiled/fake/io.pym

@@ -0,0 +1,3 @@
+class TextIOWrapper():
+    def __next__(self):
+        return 'hacked io return'

jedi/evaluate/compiled/fake/posix.pym

@@ -0,0 +1,5 @@
+def getcwd():
+    return ''
+
+def getcwdu():
+    return ''