Merge branch 'master' of github.com:davidhalter/jedi

jedi/_compatibility.py

@@ -3,7 +3,9 @@ To ensure compatibility from Python ``2.7`` - ``3.x``, a module has been
 created. Clearly there is huge need to use conforming syntax.
 """
 from __future__ import print_function
+import atexit
 import errno
+import functools
 import sys
 import os
 import re
@@ -11,6 +13,7 @@ import pkgutil
 import warnings
 import inspect
 import subprocess
+import weakref
 try:
     import importlib
 except ImportError:
@@ -635,3 +638,49 @@ def which(cmd, mode=os.F_OK | os.X_OK, path=None):
                 if _access_check(name, mode):
                     return name
     return None
+
+
+if not is_py3:
+    # Simplified backport of Python 3 weakref.finalize:
+    # https://github.com/python/cpython/blob/ded4737989316653469763230036b04513cb62b3/Lib/weakref.py#L502-L662
+    class finalize(object):
+        """Class for finalization of weakrefable objects.
+
+        finalize(obj, func, *args, **kwargs) returns a callable finalizer
+        object which will be called when obj is garbage collected. The
+        first time the finalizer is called it evaluates func(*arg, **kwargs)
+        and returns the result. After this the finalizer is dead, and
+        calling it just returns None.
+
+        When the program exits any remaining finalizers will be run.
+        """
+
+        # Finalizer objects don't have any state of their own.
+        # This ensures that they cannot be part of a ref-cycle.
+        __slots__ = ()
+        _registry = {}
+
+        def __init__(self, obj, func, *args, **kwargs):
+            info = functools.partial(func, *args, **kwargs)
+            info.weakref = weakref.ref(obj, self)
+            self._registry[self] = info
+
+        def __call__(self):
+            """Return func(*args, **kwargs) if alive."""
+            info = self._registry.pop(self, None)
+            if info:
+                return info()
+
+        @classmethod
+        def _exitfunc(cls):
+            if not cls._registry:
+                return
+            for finalizer in list(cls._registry):
+                try:
+                    finalizer(None)
+                except Exception:
+                    sys.excepthook(*sys.exc_info())
+                assert finalizer not in cls._registry
+
+    atexit.register(finalize._exitfunc)
+    weakref.finalize = finalize

jedi/api/classes.py

@@ -102,6 +102,7 @@ class BaseDefinition(object):
         to Jedi, :meth:`jedi.Script.goto_definitions` should return a list of
         definition for ``sys``, ``f``, ``C`` and ``x``.
 
+        >>> from jedi._compatibility import no_unicode_pprint
         >>> from jedi import Script
         >>> source = '''
         ... import keyword
@@ -127,9 +128,11 @@ class BaseDefinition(object):
         so that it is easy to relate the result to the source code.
 
         >>> defs = sorted(defs, key=lambda d: d.line)
-        >>> defs                           # doctest: +NORMALIZE_WHITESPACE
-        [<Definition module keyword>, <Definition class C>,
-         <Definition instance D>, <Definition def f>]
+        >>> no_unicode_pprint(defs)  # doctest: +NORMALIZE_WHITESPACE
+        [<Definition full_name='keyword', description='module keyword'>,
+         <Definition full_name='__main__.C', description='class C'>,
+         <Definition full_name='__main__.D', description='instance D'>,
+         <Definition full_name='__main__.f', description='def f'>]
 
         Finally, here is what you can get from :attr:`type`:
 
@@ -207,7 +210,7 @@ class BaseDefinition(object):
         >>> source = 'import json'
         >>> script = Script(source, path='example.py')
         >>> d = script.goto_definitions()[0]
-        >>> print(d.module_name)                       # doctest: +ELLIPSIS
+        >>> print(d.module_name)  # doctest: +ELLIPSIS
         json
         """
         return self._get_module().name.string_name
@@ -515,6 +518,7 @@ class Definition(BaseDefinition):
 
         Example:
 
+        >>> from jedi._compatibility import no_unicode_pprint
         >>> from jedi import Script
         >>> source = '''
         ... def f():
@@ -527,8 +531,9 @@ class Definition(BaseDefinition):
         >>> script = Script(source, column=3)  # line is maximum by default
         >>> defs = script.goto_definitions()
         >>> defs = sorted(defs, key=lambda d: d.line)
-        >>> defs
-        [<Definition def f>, <Definition class C>]
+        >>> no_unicode_pprint(defs)  # doctest: +NORMALIZE_WHITESPACE
+        [<Definition full_name='__main__.f', description='def f'>,
+         <Definition full_name='__main__.C', description='class C'>]
         >>> str(defs[0].description)  # strip literals in python2
         'def f'
         >>> str(defs[1].description)

jedi/evaluate/compiled/subprocess/__init__.py

@@ -12,7 +12,6 @@ import sys
 import subprocess
 import socket
 import errno
-import weakref
 import traceback
 from functools import partial
 from threading import Thread
@@ -22,7 +21,7 @@ except ImportError:
     from Queue import Queue, Empty  # python 2.7
 
 from jedi._compatibility import queue, is_py3, force_unicode, \
-    pickle_dump, pickle_load, GeneralizedPopen
+    pickle_dump, pickle_load, GeneralizedPopen, weakref
 from jedi import debug
 from jedi.cache import memoize_method
 from jedi.evaluate.compiled.subprocess import functions
@@ -37,7 +36,6 @@ _MAIN_PATH = os.path.join(os.path.dirname(__file__), '__main__.py')
 def _enqueue_output(out, queue):
     for line in iter(out.readline, b''):
         queue.put(line)
-    out.close()
 
 
 def _add_stderr_to_debug(stderr_queue):
@@ -56,6 +54,22 @@ def _get_function(name):
     return getattr(functions, name)
 
 
+def _cleanup_process(process, thread):
+    try:
+        process.kill()
+        process.wait()
+    except OSError:
+        # Raised if the process is already killed.
+        pass
+    thread.join()
+    for stream in [process.stdin, process.stdout, process.stderr]:
+        try:
+            stream.close()
+        except OSError:
+            # Raised if the stream is broken.
+            pass
+
+
 class _EvaluatorProcess(object):
     def __init__(self, evaluator):
         self._evaluator_weakref = weakref.ref(evaluator)
@@ -145,6 +159,7 @@ class CompiledSubprocess(object):
     def __init__(self, executable):
         self._executable = executable
         self._evaluator_deletion_queue = queue.deque()
+        self._cleanup_callable = lambda: None
 
     def __repr__(self):
         pid = os.getpid()
@@ -182,6 +197,12 @@ class CompiledSubprocess(object):
         )
         t.daemon = True
         t.start()
+        # Ensure the subprocess is properly cleaned up when the object
+        # is garbage collected.
+        self._cleanup_callable = weakref.finalize(self,
+                                                  _cleanup_process,
+                                                  process,
+                                                  t)
         return process
 
     def run(self, evaluator, function, args=(), kwargs={}):
@@ -202,18 +223,7 @@ class CompiledSubprocess(object):
 
     def _kill(self):
         self.is_crashed = True
-        try:
-            self._get_process().kill()
-            self._get_process().wait()
-        except (AttributeError, TypeError):
-            # If the Python process is terminating, it will remove some modules
-            # earlier than others and in general it's unclear how to deal with
-            # that so we just ignore the exceptions here.
-            pass
-
-    def __del__(self):
-        if not self.is_crashed:
-            self._kill()
+        self._cleanup_callable()
 
     def _send(self, evaluator_id, function, args=(), kwargs={}):
         if self.is_crashed:

jedi/evaluate/filters.py

@@ -359,10 +359,11 @@ def get_global_filters(evaluator, context, until_position, origin_scope):
 
     First we get the names from the function scope.
 
-    >>> no_unicode_pprint(filters[0])                    #doctest: +ELLIPSIS
+    >>> no_unicode_pprint(filters[0])  # doctest: +ELLIPSIS
     MergedFilter(<ParserTreeFilter: ...>, <GlobalNameFilter: ...>)
-    >>> sorted(str(n) for n in filters[0].values())
-    ['<TreeNameDefinition: func@(3, 4)>', '<TreeNameDefinition: x@(2, 0)>']
+    >>> sorted(str(n) for n in filters[0].values())  # doctest: +NORMALIZE_WHITESPACE
+    ['<TreeNameDefinition: string_name=func start_pos=(3, 4)>',
+     '<TreeNameDefinition: string_name=x start_pos=(2, 0)>']
     >>> filters[0]._filters[0]._until_position
     (4, 0)
     >>> filters[0]._filters[1]._until_position
@@ -380,7 +381,7 @@ def get_global_filters(evaluator, context, until_position, origin_scope):
     Finally, it yields the builtin filter, if `include_builtin` is
     true (default).
 
-    >>> list(filters[3].values())                        #doctest: +ELLIPSIS
+    >>> list(filters[3].values())  # doctest: +ELLIPSIS
     [...]
     """
     from jedi.evaluate.context.function import FunctionExecutionContext