Stop changing the global recursion limit on import (#2113)

AUTHORS.txt

@@ -69,6 +69,7 @@ Code Contributors
 - haoran3160-afk (@haoran3160-afk)
 - Eric3-jp (@Eric3-jp) (with OpenAI Codex assistance)
 - Dhanraj E (@Dhanu-dynamic) <dhanraj@labs.digiscrypt.com>
+- Charan Rathore (@charan-rathore)
 
 And a few more "anonymous" contributors.
 

jedi/api/__init__.py

@@ -7,7 +7,6 @@ Alternatively, if you don't need a custom function and are happy with printing
 debug messages to stdout, simply call :func:`set_debug_function` without
 arguments.
 """
-import sys
 from pathlib import Path
 
 import parso
@@ -21,7 +20,7 @@ from jedi.file_io import KnownContentFileIO
 from jedi.api import classes
 from jedi.api import interpreter
 from jedi.api import helpers
-from jedi.api.helpers import validate_line_column
+from jedi.api.helpers import validate_line_column, recursion_limit
 from jedi.api.completion import Completion, search_in_module
 from jedi.api.keywords import KeywordName
 from jedi.api.environment import InterpreterEnvironment
@@ -43,10 +42,6 @@ from jedi.inference.gradual.conversion import convert_names, convert_values
 from jedi.inference.gradual.utils import load_proper_stub_module
 from jedi.inference.utils import to_list
 
-# Jedi uses lots and lots of recursion. By setting this a little bit higher, we
-# can remove some "maximum recursion depth" errors.
-sys.setrecursionlimit(3000)
-
 
 class Script:
     """
@@ -97,6 +92,7 @@ class Script:
         references works well, because the right folder is searched. There are
         also ways to modify the sys path and other things.
     """
+    @recursion_limit
     def __init__(self, code=None, *, path=None, environment=None, project=None):
         self._orig_path = path
         if isinstance(path, str):
@@ -191,6 +187,7 @@ class Script:
             self._inference_state.environment,
         )
 
+    @recursion_limit
     @validate_line_column
     def complete(self, line=None, column=None, *, fuzzy=False):
         """
@@ -214,6 +211,7 @@ class Script:
             )
             return completion.complete()
 
+    @recursion_limit
     @validate_line_column
     def infer(self, line=None, column=None, *, only_stubs=False, prefer_stubs=False):
         """
@@ -259,6 +257,7 @@ class Script:
         # the API.
         return helpers.sorted_definitions(set(defs))
 
+    @recursion_limit
     @validate_line_column
     def goto(self, line=None, column=None, *, follow_imports=False, follow_builtin_imports=False,
              only_stubs=False, prefer_stubs=False):
@@ -324,6 +323,7 @@ class Script:
         """
         return self._search_func(string, all_scopes=all_scopes)
 
+    @recursion_limit
     @to_list
     def _search_func(self, string, all_scopes=False, complete=False, fuzzy=False):
         names = self._names(all_scopes=all_scopes)
@@ -353,6 +353,7 @@ class Script:
         """
         return self._search_func(string, complete=True, **kwargs)
 
+    @recursion_limit
     @validate_line_column
     def help(self, line=None, column=None):
         """
@@ -397,6 +398,7 @@ class Script:
                 return [classes.Name(self._inference_state, name)]
         return []
 
+    @recursion_limit
     @validate_line_column
     def get_references(self, line=None, column=None, **kwargs):
         """
@@ -428,6 +430,7 @@ class Script:
             return helpers.sorted_definitions(definitions)
         return _references(**kwargs)
 
+    @recursion_limit
     @validate_line_column
     def get_signatures(self, line=None, column=None):
         """
@@ -466,6 +469,7 @@ class Script:
         return [classes.Signature(self._inference_state, signature, call_details)
                 for signature in definitions.get_signatures()]
 
+    @recursion_limit
     @validate_line_column
     def get_context(self, line=None, column=None):
         """
@@ -543,6 +547,7 @@ class Script:
         finally:
             self._inference_state.is_analysis = False
 
+    @recursion_limit
     def get_names(self, **kwargs):
         """
         Returns names defined in the current file.
@@ -581,6 +586,7 @@ class Script:
         ]
         return sorted(defs, key=lambda x: x.start_pos)
 
+    @recursion_limit
     def rename(self, line=None, column=None, *, new_name):
         """
         Renames all references of the variable under the cursor.
@@ -593,6 +599,7 @@ class Script:
         definitions = self.get_references(line, column, include_builtins=False)
         return refactoring.rename(self._inference_state, definitions, new_name)
 
+    @recursion_limit
     @validate_line_column
     def extract_variable(self, line, column, *, new_name, until_line=None, until_column=None):
         """
@@ -632,6 +639,7 @@ class Script:
             new_name, (line, column), until_pos
         )
 
+    @recursion_limit
     @validate_line_column
     def extract_function(self, line, column, *, new_name, until_line=None, until_column=None):
         """
@@ -679,6 +687,7 @@ class Script:
             new_name, (line, column), until_pos
         )
 
+    @recursion_limit
     def inline(self, line=None, column=None):
         """
         Inlines a variable under the cursor. This is basically the opposite of

jedi/api/classes.py

@@ -28,7 +28,7 @@ from jedi.inference.gradual.conversion import convert_names, convert_values
 from jedi.inference.base_value import ValueSet, HasNoContext
 from jedi.api.keywords import KeywordName
 from jedi.api import completion_cache
-from jedi.api.helpers import filter_follow_imports
+from jedi.api.helpers import filter_follow_imports, recursion_limit
 
 
 def _sort_names_by_start_pos(names):
@@ -262,6 +262,7 @@ class BaseName:
             return last_leaf.end_pos
         return definition.end_pos
 
+    @recursion_limit
     def docstring(self, raw=False, fast=True):
         r"""
         Return a document string for this completion object.
@@ -422,6 +423,7 @@ class BaseName:
         return tree_name.is_definition() and tree_name.parent.type == 'trailer'
 
     @debug.increase_indent_cm('goto on name')
+    @recursion_limit
     def goto(self, *, follow_imports=False, follow_builtin_imports=False,
              only_stubs=False, prefer_stubs=False):
 
@@ -452,6 +454,7 @@ class BaseName:
                 for n in names]
 
     @debug.increase_indent_cm('infer on name')
+    @recursion_limit
     def infer(self, *, only_stubs=False, prefer_stubs=False):
         """
         Like :meth:`.Script.infer`, it can be useful to understand which type
@@ -567,6 +570,7 @@ class BaseName:
         names = convert_names([self._name], prefer_stubs=True)
         return [sig for name in names for sig in name.infer().get_signatures()]
 
+    @recursion_limit
     def get_signatures(self):
         """
         Returns all potential signatures for a function or a class. Multiple
@@ -579,6 +583,7 @@ class BaseName:
             for s in self._get_signatures()
         ]
 
+    @recursion_limit
     def execute(self):
         """
         Uses type inference to "execute" this identifier and returns the
@@ -588,6 +593,7 @@ class BaseName:
         """
         return _values_to_definitions(self._name.infer().execute_with_values())
 
+    @recursion_limit
     def get_type_hint(self):
         """
         Returns type hints like ``Iterable[int]`` or ``Union[int, str]``.
@@ -670,6 +676,7 @@ class Completion(BaseName):
         """
         return self._complete(False)
 
+    @recursion_limit
     def docstring(self, raw=False, fast=True):
         """
         Documented under :meth:`BaseName.docstring`.
@@ -752,6 +759,7 @@ class Name(BaseName):
         super().__init__(inference_state, definition)
 
     @memoize_method
+    @recursion_limit
     def defined_names(self):
         """
         List sub-definitions (e.g., methods in class).
@@ -797,6 +805,7 @@ class BaseSignature(Name):
         self._signature = signature
 
     @property
+    @recursion_limit
     def params(self):
         """
         Returns definitions for all parameters that a signature defines.
@@ -807,6 +816,7 @@ class BaseSignature(Name):
         return [ParamName(self._inference_state, n)
                 for n in self._signature.get_param_names(resolve_stars=True)]
 
+    @recursion_limit
     def to_string(self):
         """
         Returns a text representation of the signature. This could for example
@@ -858,6 +868,7 @@ class Signature(BaseSignature):
 
 
 class ParamName(Name):
+    @recursion_limit
     def infer_default(self):
         """
         Returns default values like the ``1`` of ``def foo(x=1):``.
@@ -866,6 +877,7 @@ class ParamName(Name):
         """
         return _values_to_definitions(self._name.infer_default())
 
+    @recursion_limit
     def infer_annotation(self, **kwargs):
         """
         :param execute_annotation: Default True; If False, values are not
@@ -874,6 +886,7 @@ class ParamName(Name):
         """
         return _values_to_definitions(self._name.infer_annotation(ignore_stars=True, **kwargs))
 
+    @recursion_limit
     def to_string(self):
         """
         Returns a simple representation of a param, like

jedi/api/helpers.py

@@ -2,7 +2,9 @@
 Helpers for the API
 """
 import re
+import sys
 from collections import namedtuple
+from contextlib import contextmanager
 from textwrap import dedent
 from itertools import chain
 from functools import wraps
@@ -21,6 +23,48 @@ from jedi.parser_utils import get_parent_scope
 
 CompletionParts = namedtuple('CompletionParts', ['path', 'has_dot', 'name'])
 
+RECURSION_LIMIT = 3000
+
+
+@contextmanager
+def higher_recursion_limit():
+    """Jedi needs a higher recursion limit for parsing and inference. Raise it
+    only temporarily, so importing Jedi does not change global interpreter
+    state and a user-defined higher limit is never lowered."""
+    old_limit = sys.getrecursionlimit()
+    if old_limit >= RECURSION_LIMIT:
+        yield
+    else:
+        sys.setrecursionlimit(RECURSION_LIMIT)
+        try:
+            yield
+        finally:
+            sys.setrecursionlimit(old_limit)
+
+
+def recursion_limit(func):
+    @wraps(func)
+    def wrapper(*args, **kwargs):
+        with higher_recursion_limit():
+            return func(*args, **kwargs)
+    return wrapper
+
+
+def recursion_limit_generator(func):
+    @wraps(func)
+    def wrapper(*args, **kwargs):
+        with higher_recursion_limit():
+            iterator = iter(func(*args, **kwargs))
+        while True:
+            with higher_recursion_limit():
+                try:
+                    value = next(iterator)
+                except StopIteration:
+                    return
+            # Restore the caller's limit before yielding a result.
+            yield value
+    return wrapper
+
 
 def _start_match(string, like_name):
     return string.startswith(like_name)

jedi/api/project.py

@@ -15,7 +15,7 @@ from jedi import debug
 from jedi.api.environment import get_cached_default_environment, create_environment
 from jedi.api.exceptions import WrongVersion
 from jedi.api.completion import search_in_module
-from jedi.api.helpers import split_search_string, get_module_names
+from jedi.api.helpers import split_search_string, get_module_names, recursion_limit_generator
 from jedi.inference.imports import load_module_from_path, \
     load_namespace_from_path, iter_module_names
 from jedi.inference.sys_path import discover_buildout_paths
@@ -293,6 +293,7 @@ class Project:
         """
         return self._search_func(string, complete=True, **kwargs)
 
+    @recursion_limit_generator
     @_try_to_skip_duplicates
     def _search_func(self, string, complete=False, all_scopes=False):
         # Using a Script is they easiest way to get an empty module context.

test/test_api/test_api.py

@@ -3,6 +3,8 @@ Test all things related to the ``jedi.api`` module.
 """
 
 import os
+import subprocess
+import sys
 from textwrap import dedent
 
 import pytest
@@ -425,3 +427,90 @@ def test_infer_after_parentheses(Script, code, column, expected):
         assert completions == []
     else:
         assert [c.name for c in completions] == [expected]
+
+
+def test_import_does_not_change_recursion_limit():
+    # Regression test for #1925: importing jedi must not lower a higher
+    # user-defined limit, nor raise a lower one.
+    for limit in (1200, 5000):
+        code = ('import sys; sys.setrecursionlimit(%d); import jedi; '
+                'print(sys.getrecursionlimit())' % limit)
+        output = subprocess.check_output([sys.executable, '-c', code])
+        assert output.strip() == str(limit).encode()
+
+
+def test_recursion_limit_is_scoped_and_restored():
+    from jedi.api.helpers import higher_recursion_limit, RECURSION_LIMIT
+
+    old_limit = sys.getrecursionlimit()
+    sys.setrecursionlimit(1000)
+    try:
+        with higher_recursion_limit():
+            assert sys.getrecursionlimit() == RECURSION_LIMIT
+        assert sys.getrecursionlimit() == 1000
+        sys.setrecursionlimit(5000)
+        with higher_recursion_limit():
+            assert sys.getrecursionlimit() == 5000
+        assert sys.getrecursionlimit() == 5000
+    finally:
+        sys.setrecursionlimit(old_limit)
+
+
+def test_deep_code_works_with_low_recursion_limit(Script):
+    old_limit = sys.getrecursionlimit()
+    sys.setrecursionlimit(1000)
+    try:
+        code = 'x = ' + '(' * 200 + '1' + ')' * 200 + '\nx.'
+        assert Script(code).complete(2, 2)
+        assert sys.getrecursionlimit() == 1000
+    finally:
+        sys.setrecursionlimit(old_limit)
+
+
+@pytest.mark.parametrize('method', [
+    'infer', 'get_type_hint', 'get_signatures', 'execute', 'defined_names',
+])
+def test_name_inference_recursion_limit(Script, method):
+    old_limit = sys.getrecursionlimit()
+    sys.setrecursionlimit(1000)
+    try:
+        name = Script('x = ' + '(' * 200 + '1' + ')' * 200).get_names()[0]
+        getattr(name, method)()
+        assert sys.getrecursionlimit() == 1000
+    finally:
+        sys.setrecursionlimit(old_limit)
+
+
+@pytest.mark.parametrize('method', ['search', 'complete_search'])
+def test_project_search_recursion_limit(tmp_path, method):
+    from jedi import Project
+
+    (tmp_path / 'sample.py').write_text('x = ' + '(' * 200 + '1' + ')' * 200)
+    old_limit = sys.getrecursionlimit()
+    sys.setrecursionlimit(1000)
+    try:
+        results = getattr(Project(tmp_path), method)('sample.x.real')
+        assert sys.getrecursionlimit() == 1000
+        assert next(results).name == 'real'
+        assert sys.getrecursionlimit() == 1000
+        results.close()
+        assert sys.getrecursionlimit() == 1000
+    finally:
+        sys.setrecursionlimit(old_limit)
+
+
+def test_recursion_limit_restored_after_error():
+    from jedi.api.helpers import recursion_limit
+
+    @recursion_limit
+    def fail():
+        raise ValueError('test')
+
+    old_limit = sys.getrecursionlimit()
+    sys.setrecursionlimit(1000)
+    try:
+        with pytest.raises(ValueError, match='test'):
+            fail()
+        assert sys.getrecursionlimit() == 1000
+    finally:
+        sys.setrecursionlimit(old_limit)