From 744662d09693cd81fec159101f1d50cfc68af52f Mon Sep 17 00:00:00 2001 From: micbou Date: Thu, 28 Feb 2019 15:40:24 +0100 Subject: [PATCH] Fix resource warnings --- jedi/_compatibility.py | 49 +++++++++++++++++++ jedi/evaluate/compiled/subprocess/__init__.py | 40 +++++++++------ test/run.py | 6 ++- 3 files changed, 78 insertions(+), 17 deletions(-) diff --git a/jedi/_compatibility.py b/jedi/_compatibility.py index 8ce0e685..f6c1422f 100644 --- a/jedi/_compatibility.py +++ b/jedi/_compatibility.py @@ -3,7 +3,9 @@ To ensure compatibility from Python ``2.7`` - ``3.x``, a module has been created. Clearly there is huge need to use conforming syntax. """ from __future__ import print_function +import atexit import errno +import functools import sys import os import re @@ -11,6 +13,7 @@ import pkgutil import warnings import inspect import subprocess +import weakref try: import importlib except ImportError: @@ -635,3 +638,49 @@ def which(cmd, mode=os.F_OK | os.X_OK, path=None): if _access_check(name, mode): return name return None + + +if not is_py3: + # Simplified backport of Python 3 weakref.finalize: + # https://github.com/python/cpython/blob/ded4737989316653469763230036b04513cb62b3/Lib/weakref.py#L502-L662 + class finalize(object): + """Class for finalization of weakrefable objects. + + finalize(obj, func, *args, **kwargs) returns a callable finalizer + object which will be called when obj is garbage collected. The + first time the finalizer is called it evaluates func(*arg, **kwargs) + and returns the result. After this the finalizer is dead, and + calling it just returns None. + + When the program exits any remaining finalizers will be run. + """ + + # Finalizer objects don't have any state of their own. + # This ensures that they cannot be part of a ref-cycle. + __slots__ = () + _registry = {} + + def __init__(self, obj, func, *args, **kwargs): + info = functools.partial(func, *args, **kwargs) + info.weakref = weakref.ref(obj, self) + self._registry[self] = info + + def __call__(self): + """Return func(*args, **kwargs) if alive.""" + info = self._registry.pop(self, None) + if info: + return info() + + @classmethod + def _exitfunc(cls): + if not cls._registry: + return + for finalizer in list(cls._registry): + try: + finalizer(None) + except Exception: + sys.excepthook(*sys.exc_info()) + assert finalizer not in cls._registry + + atexit.register(finalize._exitfunc) + weakref.finalize = finalize diff --git a/jedi/evaluate/compiled/subprocess/__init__.py b/jedi/evaluate/compiled/subprocess/__init__.py index f8dc8f20..dea2f66d 100644 --- a/jedi/evaluate/compiled/subprocess/__init__.py +++ b/jedi/evaluate/compiled/subprocess/__init__.py @@ -12,7 +12,6 @@ import sys import subprocess import socket import errno -import weakref import traceback from functools import partial from threading import Thread @@ -22,7 +21,7 @@ except ImportError: from Queue import Queue, Empty # python 2.7 from jedi._compatibility import queue, is_py3, force_unicode, \ - pickle_dump, pickle_load, GeneralizedPopen + pickle_dump, pickle_load, GeneralizedPopen, weakref from jedi import debug from jedi.cache import memoize_method from jedi.evaluate.compiled.subprocess import functions @@ -37,7 +36,6 @@ _MAIN_PATH = os.path.join(os.path.dirname(__file__), '__main__.py') def _enqueue_output(out, queue): for line in iter(out.readline, b''): queue.put(line) - out.close() def _add_stderr_to_debug(stderr_queue): @@ -56,6 +54,22 @@ def _get_function(name): return getattr(functions, name) +def _cleanup_process(process, thread): + try: + process.kill() + process.wait() + except OSError: + # Raised if the process is already killed. + pass + thread.join() + for stream in [process.stdin, process.stdout, process.stderr]: + try: + stream.close() + except OSError: + # Raised if the stream is broken. + pass + + class _EvaluatorProcess(object): def __init__(self, evaluator): self._evaluator_weakref = weakref.ref(evaluator) @@ -145,6 +159,7 @@ class CompiledSubprocess(object): def __init__(self, executable): self._executable = executable self._evaluator_deletion_queue = queue.deque() + self._cleanup_callable = lambda: None def __repr__(self): pid = os.getpid() @@ -182,6 +197,12 @@ class CompiledSubprocess(object): ) t.daemon = True t.start() + # Ensure the subprocess is properly cleaned up when the object + # is garbage collected. + self._cleanup_callable = weakref.finalize(self, + _cleanup_process, + process, + t) return process def run(self, evaluator, function, args=(), kwargs={}): @@ -202,18 +223,7 @@ class CompiledSubprocess(object): def _kill(self): self.is_crashed = True - try: - self._get_process().kill() - self._get_process().wait() - except (AttributeError, TypeError): - # If the Python process is terminating, it will remove some modules - # earlier than others and in general it's unclear how to deal with - # that so we just ignore the exceptions here. - pass - - def __del__(self): - if not self.is_crashed: - self._kill() + self._cleanup_callable() def _send(self, evaluator_id, function, args=(), kwargs={}): if self.is_crashed: diff --git a/test/run.py b/test/run.py index 6ed0dbec..c5eb0330 100755 --- a/test/run.py +++ b/test/run.py @@ -357,9 +357,11 @@ def collect_dir_tests(base_dir, test_files, check_thirdparty=False): path = os.path.join(base_dir, f_name) if is_py3: - source = open(path, encoding='utf-8').read() + with open(path, encoding='utf-8') as f: + source = f.read() else: - source = unicode(open(path).read(), 'UTF-8') + with open(path) as f: + source = unicode(f.read(), 'UTF-8') for case in collect_file_tests(path, StringIO(source), lines_to_execute):