Use split_lines and python_bytes_to_unicode directly.

CONTRIBUTING.md

@@ -1,28 +1,8 @@
-Pull Requests are great (on the **dev** branch)! Readme/Documentation changes
-are ok in the master branch.
+Pull Requests are great.
 
  1. Fork the Repo on github.
  2. If you are adding functionality or fixing a bug, please add a test!
  3. Add your name to AUTHORS.txt
- 4. Push to your fork and submit a **pull request to the dev branch**.
-
-My **master** branch is a 100% stable (should be). I only push to it after I am
-certain that things are working out. Many people are using Jedi directly from
-the github master branch.
+ 4. Push to your fork and submit a pull request.
 
 **Try to use the PEP8 style guide.**
-
-
-Changing Issues to Pull Requests (Github)
------------------------------------------
-
-If you have have previously filed a GitHub issue and want to contribute code
-that addresses that issue, we prefer it if you use
-[hub](https://github.com/github/hub) to convert your existing issue to a pull
-request. To do that, first push the changes to a separate branch in your fork
-and then issue the following command:
-
-    hub pull-request -b davidhalter:dev -i <issue-number> -h <your-github-username>:<your-branch-name>
-
-It's no strict requirement though, if you don't have hub installed or prefer to
-use the web interface, then feel free to post a traditional pull request.

README.rst

@@ -12,7 +12,7 @@ Jedi - an awesome autocompletion/static analysis library for Python
 
 
 *If you have specific questions, please add an issue or ask on* `stackoverflow
-<https://stackoverflow.com>`_ *with the label* ``python-jedi``.
+<https://stackoverflow.com/questions/tagged/python-jedi>`_ *with the label* ``python-jedi``.
 
 
 Jedi is a static analysis tool for Python that can be used in IDEs/editors. Its

deploy-master.sh

@@ -0,0 +1,52 @@
+#!/usr/bin/env bash
+# The script creates a separate folder in build/ and creates tags there, pushes
+# them and then uploads the package to PyPI.
+
+set -eu -o pipefail
+
+BASE_DIR=$(dirname $(readlink -f "$0"))
+cd $BASE_DIR
+
+git fetch --tags
+
+PROJECT_NAME=jedi
+BRANCH=master
+BUILD_FOLDER=build
+
+[ -d $BUILD_FOLDER ] || mkdir $BUILD_FOLDER
+# Remove the previous deployment first.
+# Checkout the right branch
+cd $BUILD_FOLDER
+rm -rf $PROJECT_NAME
+git clone .. $PROJECT_NAME
+cd $PROJECT_NAME
+git checkout $BRANCH
+
+# Test first.
+tox
+
+# Create tag
+tag=v$(python -c "import $PROJECT_NAME; print($PROJECT_NAME.__version__)")
+
+master_ref=$(git show-ref -s heads/$BRANCH)
+tag_ref=$(git show-ref -s $tag || true)
+if [[ $tag_ref ]]; then
+    if [[ $tag_ref != $master_ref ]]; then
+        echo 'Cannot tag something that has already been tagged with another commit.'
+        exit 1
+    fi
+else
+    git tag $tag
+    git push --tags
+fi
+
+# Package and upload to PyPI
+#rm -rf dist/ - Not needed anymore, because the folder is never reused.
+echo `pwd`
+python setup.py sdist bdist_wheel
+# Maybe do a pip install twine before.
+twine upload dist/*
+
+cd $BASE_DIR
+# Back in the development directory fetch tags.
+git fetch --tags

jedi/api/__init__.py

@@ -15,7 +15,7 @@ import sys
 
 import parso
 from parso.python import tree
-from parso.utils import python_bytes_to_unicode, split_lines
+from parso import python_bytes_to_unicode, split_lines
 
 from jedi.parser_utils import get_executable_nodes, get_statement_of_position
 from jedi import debug

jedi/api/helpers.py

@@ -7,7 +7,7 @@ from textwrap import dedent
 
 from parso.python.parser import Parser
 from parso.python import tree
-from parso.utils import split_lines
+from parso import split_lines
 
 from jedi._compatibility import u
 from jedi.evaluate.helpers import evaluate_call_of_leaf

jedi/api/interpreter.py

@@ -8,6 +8,11 @@ from jedi.evaluate.compiled import mixed
 from jedi.evaluate.context import Context
 
 
+class NamespaceObject(object):
+    def __init__(self, dct):
+        self.__dict__ = dct
+
+
 class MixedModuleContext(Context):
     resets_positions = True
     type = 'mixed_module'
@@ -16,7 +21,7 @@ class MixedModuleContext(Context):
         self.evaluator = evaluator
         self._namespaces = namespaces
 
-        self._namespace_objects = [type('jedi_namespace', (), n) for n in namespaces]
+        self._namespace_objects = [NamespaceObject(n) for n in namespaces]
         self._module_context = ModuleContext(evaluator, tree_module, path=path)
         self.tree_node = tree_module
 

jedi/api/keywords.py

@@ -76,12 +76,16 @@ class KeywordName(AbstractNameDefinition):
     api_type = 'keyword'
 
     def __init__(self, evaluator, name):
+        self.evaluator = evaluator
         self.string_name = name
         self.parent_context = evaluator.BUILTINS
 
     def eval(self):
         return set()
 
+    def infer(self):
+        return [Keyword(self.evaluator, self.string_name, (0, 0))]
+
 
 class Keyword(object):
     api_type = 'keyword'
@@ -100,9 +104,8 @@ class Keyword(object):
         """ For a `parsing.Name` like comparision """
         return [self.name]
 
-    @property
-    def docstr(self):
-        return imitate_pydoc(self.name)
+    def py__doc__(self, include_call_signature=False):
+        return imitate_pydoc(self.name.string_name)
 
     def __repr__(self):
         return '<%s: %s>' % (type(self).__name__, self.name)
@@ -136,6 +139,6 @@ def imitate_pydoc(string):
         return ''
 
     try:
-        return pydoc_topics.topics[label] if pydoc_topics else ''
+        return pydoc_topics.topics[label].strip() if pydoc_topics else ''
     except KeyError:
         return ''

jedi/evaluate/compiled/__init__.py

@@ -5,6 +5,7 @@ import inspect
 import re
 import sys
 import os
+import types
 from functools import partial
 
 from jedi._compatibility import builtins as _builtins, unicode
@@ -13,6 +14,7 @@ from jedi.cache import underscore_memoization, memoize_method
 from jedi.evaluate.filters import AbstractFilter, AbstractNameDefinition, \
     ContextNameMixin
 from jedi.evaluate.context import Context, LazyKnownContext
+from jedi.evaluate.compiled.getattr_static import  getattr_static
 from . import fake
 
 
@@ -22,6 +24,23 @@ if os.path.altsep is not None:
 _path_re = re.compile('(?:\.[^{0}]+|[{0}]__init__\.py)$'.format(re.escape(_sep)))
 del _sep
 
+# Those types don't exist in typing.
+MethodDescriptorType = type(str.replace)
+WrapperDescriptorType = type(set.__iter__)
+# `object.__subclasshook__` is an already executed descriptor.
+object_class_dict = type.__dict__["__dict__"].__get__(object)
+ClassMethodDescriptorType = type(object_class_dict['__subclasshook__'])
+
+ALLOWED_DESCRIPTOR_ACCESS = (
+    types.FunctionType,
+    types.GetSetDescriptorType,
+    types.MemberDescriptorType,
+    MethodDescriptorType,
+    WrapperDescriptorType,
+    ClassMethodDescriptorType,
+    staticmethod,
+    classmethod,
+)
 
 class CheckAttribute(object):
     """Raises an AttributeError if the attribute X isn't available."""
@@ -297,16 +316,17 @@ class CompiledObjectFilter(AbstractFilter):
         name = str(name)
         obj = self._compiled_object.obj
         try:
-            getattr(obj, name)
-            if self._is_instance and name not in dir(obj):
-                return []
+            attr, is_get_descriptor = getattr_static(obj, name)
         except AttributeError:
             return []
-        except Exception:
-            # This is a bit ugly. We're basically returning this to make
-            # lookups possible without having the actual attribute. However
-            # this makes proper completion possible.
-            return [EmptyCompiledName(self._evaluator, name)]
+        else:
+            if is_get_descriptor \
+                    and not type(attr) in ALLOWED_DESCRIPTOR_ACCESS:
+                # In case of descriptors that have get methods we cannot return
+                # it's value, because that would mean code execution.
+                return [EmptyCompiledName(self._evaluator, name)]
+            if self._is_instance and name not in dir(obj):
+                return []
         return [self._create_name(name)]
 
     def values(self):

jedi/evaluate/compiled/getattr_static.py

@@ -0,0 +1,175 @@
+"""
+A static version of getattr.
+This is a backport of the Python 3 code with a little bit of additional
+information returned to enable Jedi to make decisions.
+"""
+
+import types
+
+from jedi._compatibility import py_version
+
+_sentinel = object()
+
+def _check_instance(obj, attr):
+    instance_dict = {}
+    try:
+        instance_dict = object.__getattribute__(obj, "__dict__")
+    except AttributeError:
+        pass
+    return dict.get(instance_dict, attr, _sentinel)
+
+
+def _check_class(klass, attr):
+    for entry in _static_getmro(klass):
+        if _shadowed_dict(type(entry)) is _sentinel:
+            try:
+                return entry.__dict__[attr]
+            except KeyError:
+                pass
+    return _sentinel
+
+def _is_type(obj):
+    try:
+        _static_getmro(obj)
+    except TypeError:
+        return False
+    return True
+
+
+def _shadowed_dict_newstyle(klass):
+    dict_attr = type.__dict__["__dict__"]
+    for entry in _static_getmro(klass):
+        try:
+            class_dict = dict_attr.__get__(entry)["__dict__"]
+        except KeyError:
+            pass
+        else:
+            if not (type(class_dict) is types.GetSetDescriptorType and
+                    class_dict.__name__ == "__dict__" and
+                    class_dict.__objclass__ is entry):
+                return class_dict
+    return _sentinel
+
+
+def _static_getmro_newstyle(klass):
+    return type.__dict__['__mro__'].__get__(klass)
+
+
+if py_version >= 30:
+    _shadowed_dict = _shadowed_dict_newstyle
+    _get_type = type
+    _static_getmro = _static_getmro_newstyle
+else:
+    def _shadowed_dict(klass):
+        """
+        In Python 2 __dict__ is not overwritable:
+
+            class Foo(object): pass
+            setattr(Foo, '__dict__', 4)
+
+            Traceback (most recent call last):
+              File "<stdin>", line 1, in <module>
+            TypeError: __dict__ must be a dictionary object
+
+        It applies to both newstyle and oldstyle classes:
+
+            class Foo(object): pass
+            setattr(Foo, '__dict__', 4)
+            Traceback (most recent call last):
+              File "<stdin>", line 1, in <module>
+            AttributeError: attribute '__dict__' of 'type' objects is not writable
+
+        It also applies to instances of those objects. However to keep things
+        straight forward, newstyle classes always use the complicated way of
+        accessing it while oldstyle classes just use getattr.
+        """
+        if type(klass) is _oldstyle_class_type:
+            return getattr(klass, '__dict__', _sentinel)
+        return _shadowed_dict_newstyle(klass)
+
+    class _OldStyleClass():
+        pass
+
+    _oldstyle_instance_type = type(_OldStyleClass())
+    _oldstyle_class_type = type(_OldStyleClass)
+
+    def _get_type(obj):
+        type_ = object.__getattribute__(obj, '__class__')
+        if type_ is _oldstyle_instance_type:
+            # Somehow for old style classes we need to access it directly.
+            return obj.__class__
+        return type_
+
+    def _static_getmro(klass):
+        if type(klass) is _oldstyle_class_type:
+            def oldstyle_mro(klass):
+                """
+                Oldstyle mro is a really simplistic way of look up mro:
+                https://stackoverflow.com/questions/54867/what-is-the-difference-between-old-style-and-new-style-classes-in-python
+                """
+                yield klass
+                for base in klass.__bases__:
+                    for yield_from in oldstyle_mro(base):
+                        yield yield_from
+
+            return oldstyle_mro(klass)
+
+        return _static_getmro_newstyle(klass)
+
+
+def _safe_hasattr(obj, name):
+    return _check_class(_get_type(obj), name) is not _sentinel
+
+
+def _safe_is_data_descriptor(obj):
+    return (_safe_hasattr(obj, '__set__') or _safe_hasattr(obj, '__delete__'))
+
+
+def getattr_static(obj, attr, default=_sentinel):
+    """Retrieve attributes without triggering dynamic lookup via the
+       descriptor protocol,  __getattr__ or __getattribute__.
+
+       Note: this function may not be able to retrieve all attributes
+       that getattr can fetch (like dynamically created attributes)
+       and may find attributes that getattr can't (like descriptors
+       that raise AttributeError). It can also return descriptor objects
+       instead of instance members in some cases. See the
+       documentation for details.
+
+       Returns a tuple `(attr, is_get_descriptor)`. is_get_descripter means that
+       the attribute is a descriptor that has a `__get__` attribute.
+    """
+    instance_result = _sentinel
+    if not _is_type(obj):
+        klass = _get_type(obj)
+        dict_attr = _shadowed_dict(klass)
+        if (dict_attr is _sentinel or
+            type(dict_attr) is types.MemberDescriptorType):
+            instance_result = _check_instance(obj, attr)
+    else:
+        klass = obj
+
+    klass_result = _check_class(klass, attr)
+
+    if instance_result is not _sentinel and klass_result is not _sentinel:
+        if _safe_hasattr(klass_result, '__get__') \
+                and _safe_is_data_descriptor(klass_result):
+            # A get/set descriptor has priority over everything.
+            return klass_result, True
+
+    if instance_result is not _sentinel:
+        return instance_result, False
+    if klass_result is not _sentinel:
+        return klass_result, _safe_hasattr(klass_result, '__get__')
+
+    if obj is klass:
+        # for types we check the metaclass too
+        for entry in _static_getmro(type(klass)):
+            if _shadowed_dict(type(entry)) is _sentinel:
+                try:
+                    return entry.__dict__[attr], False
+                except KeyError:
+                    pass
+    if default is not _sentinel:
+        return default, False
+    raise AttributeError(attr)

jedi/evaluate/imports.py

@@ -19,7 +19,7 @@ import sys
 from parso.python import tree
 from parso.tree import search_ancestor
 from parso.cache import parser_cache
-from parso.utils import python_bytes_to_unicode
+from parso import python_bytes_to_unicode
 
 from jedi._compatibility import find_module, unicode, ImplicitNSInfo
 from jedi import debug

jedi/evaluate/instance.py

@@ -197,6 +197,11 @@ class CompiledInstance(AbstractInstanceContext):
 
 
 class TreeInstance(AbstractInstanceContext):
+    def __init__(self, evaluator, parent_context, class_context, var_args):
+        super(TreeInstance, self).__init__(evaluator, parent_context,
+                                           class_context, var_args)
+        self.tree_node = class_context.tree_node
+
     @property
     def name(self):
         return filters.ContextName(self, self.class_context.name.tree_name)

jedi/evaluate/representation.py

@@ -44,7 +44,7 @@ import re
 from itertools import chain
 
 from parso.python import tree
-from parso.utils import python_bytes_to_unicode
+from parso import python_bytes_to_unicode
 
 from jedi._compatibility import use_metaclass
 from jedi import debug

jedi/refactoring.py

@@ -15,7 +15,7 @@ following functions (sometimes bug-prone):
 import difflib
 
 from jedi import common
-from parso.utils import python_bytes_to_unicode, split_lines
+from parso import python_bytes_to_unicode, split_lines
 from jedi.evaluate import helpers
 
 
@@ -29,7 +29,7 @@ class Refactoring(object):
     def old_files(self):
         dct = {}
         for old_path, (new_path, old_l, new_l) in self.change_dct.items():
-            dct[new_path] = '\n'.join(new_l)
+            dct[old_path] = '\n'.join(old_l)
         return dct
 
     def new_files(self):

jedi/utils.py

@@ -11,7 +11,7 @@ import re
 import os
 import sys
 
-from parso.utils import split_lines
+from parso import split_lines
 
 from jedi import Interpreter
 from jedi.api.helpers import get_on_completion_name

test/test_api/test_interpreter.py

@@ -2,7 +2,6 @@
 Tests of ``jedi.api.Interpreter``.
 """
 
-from ..helpers import TestCase
 import jedi
 from jedi._compatibility import is_py33
 from jedi.evaluate.compiled import mixed
@@ -178,16 +177,37 @@ def test_getitem_side_effects():
     _assert_interpreter_complete('foo[0].', locals(), [])
 
 
-def test_property_error():
+def test_property_error_oldstyle():
+    lst = []
     class Foo3():
         @property
         def bar(self):
+            lst.append(1)
             raise ValueError
 
     foo = Foo3()
     _assert_interpreter_complete('foo.bar', locals(), ['bar'])
     _assert_interpreter_complete('foo.bar.baz', locals(), [])
 
+    # There should not be side effects
+    assert lst == []
+
+
+def test_property_error_newstyle():
+    lst = []
+    class Foo3(object):
+        @property
+        def bar(self):
+            lst.append(1)
+            raise ValueError
+
+    foo = Foo3()
+    _assert_interpreter_complete('foo.bar', locals(), ['bar'])
+    _assert_interpreter_complete('foo.bar.baz', locals(), [])
+
+    # There should not be side effects
+    assert lst == []
+
 
 def test_param_completion():
     def foo(bar):

test/test_evaluate/test_docstring.py

@@ -20,7 +20,22 @@ class TestDocstring(unittest.TestCase):
         def func():
             '''Docstring of `func`.'''
         func""").goto_definitions()
-        self.assertEqual(defs[0].docstring(raw=True), 'Docstring of `func`.')
+        self.assertEqual(defs[0].docstring(), 'func()\n\nDocstring of `func`.')
+
+    def test_class_doc(self):
+        defs = jedi.Script("""
+        class TestClass():
+            '''Docstring of `TestClass`.'''
+        TestClass""").goto_definitions()
+        self.assertEqual(defs[0].docstring(), 'Docstring of `TestClass`.')
+
+    def test_instance_doc(self):
+        defs = jedi.Script("""
+        class TestClass():
+            '''Docstring of `TestClass`.'''
+        tc = TestClass()
+        tc""").goto_definitions()
+        self.assertEqual(defs[0].docstring(), 'Docstring of `TestClass`.')
 
     @unittest.skip('need evaluator class for that')
     def test_attribute_docstring(self):
@@ -28,7 +43,7 @@ class TestDocstring(unittest.TestCase):
         x = None
         '''Docstring of `x`.'''
         x""").goto_definitions()
-        self.assertEqual(defs[0].docstring(raw=True), 'Docstring of `x`.')
+        self.assertEqual(defs[0].docstring(), 'Docstring of `x`.')
 
     @unittest.skip('need evaluator class for that')
     def test_multiple_docstrings(self):
@@ -38,7 +53,7 @@ class TestDocstring(unittest.TestCase):
         x = func
         '''Docstring of `x`.'''
         x""").goto_definitions()
-        docs = [d.docstring(raw=True) for d in defs]
+        docs = [d.docstring() for d in defs]
         self.assertEqual(docs, ['Original docstring.', 'Docstring of `x`.'])
 
     def test_completion(self):
@@ -105,6 +120,10 @@ class TestDocstring(unittest.TestCase):
         assert '__init__' in names
         assert 'mro' not in names  # Exists only for types.
 
+    def test_docstring_keyword(self):
+        completions = jedi.Script('assert').completions()
+        self.assertIn('assert', completions[0].docstring())
+
     @unittest.skipIf(numpydoc_unavailable, 'numpydoc module is unavailable')
     def test_numpydoc_docstring(self):
         s = dedent('''

test/test_parso_integration/test_parser_utils.py

@@ -1,5 +1,4 @@
 # -*- coding: utf-8 -*-
-
 from jedi._compatibility import is_py3
 from jedi import parser_utils
 from parso import parse