Files bigger than one MB (about 20kLOC) get cropped to avoid getting stuck completely

Fixes #843

CHANGELOG.rst

@@ -3,12 +3,14 @@
 Changelog
 ---------
 
-0.15.2 (2019-08-13)
+0.15.2 (2019--)
 +++++++++++++++++++
 
 - Call signatures are now detected a lot better
 - Many small Bugfixes
 - A big refactoring around contexts/values
+- Files bigger than one MB (about 20kLOC) get cropped to avoid getting
+  stuck completely.
 
 0.15.1 (2019-08-13)
 +++++++++++++++++++

jedi/inference/__init__.py

@@ -67,6 +67,7 @@ from parso import python_bytes_to_unicode
 from jedi.file_io import FileIO
 
 from jedi import debug
+from jedi import settings
 from jedi.inference import imports
 from jedi.inference import recursion
 from jedi.inference.cache import inference_state_function_cache
@@ -105,7 +106,6 @@ class InferenceState(object):
         self.allow_descriptor_getattr = False
 
         self.reset_recursion_limitations()
-        self.allow_different_encoding = True
 
     def import_module(self, import_names, parent_module_value=None,
                       sys_path=None, prefer_stubs=True):
@@ -181,12 +181,15 @@ class InferenceState(object):
 
     def parse_and_get_code(self, code=None, path=None, encoding='utf-8',
                            use_latest_grammar=False, file_io=None, **kwargs):
-        if self.allow_different_encoding:
-            if code is None:
-                if file_io is None:
-                    file_io = FileIO(path)
-                code = file_io.read()
-            code = python_bytes_to_unicode(code, encoding=encoding, errors='replace')
+        if code is None:
+            if file_io is None:
+                file_io = FileIO(path)
+            code = file_io.read()
+        # We cannot just use parso, because it doesn't use errors='replace'.
+        code = python_bytes_to_unicode(code, encoding=encoding, errors='replace')
+
+        if len(code) > settings._cropped_file_size:
+            code = code[:settings._cropped_file_size]
 
         grammar = self.latest_grammar if use_latest_grammar else self.grammar
         return grammar.parse(code=code, path=path, file_io=file_io, **kwargs), code

jedi/settings.py

@@ -112,6 +112,14 @@ something has been changed e.g. to a function. If this happens, only the
 function is being reparsed.
 """
 
+_cropped_file_size = 10e6  # 1 Megabyte
+"""
+Jedi gets extremely slow if the file size exceed a few thousand lines.
+To avoid getting stuck completely Jedi crops the file this point.
+
+One megabyte of typical Python code equals about 20'000 lines of code.
+"""
+
 # ----------------
 # dynamic stuff
 # ----------------

test/test_settings.py

@@ -32,3 +32,20 @@ def test_additional_dynamic_modules(monkeypatch, Script):
         ['/foo/bar/jedi_not_existing_file.py']
     )
     assert not Script('def some_func(f):\n f.').completions()
+
+
+def test_cropped_file_size(monkeypatch, names, Script):
+    code = 'class Foo(): pass\n'
+    monkeypatch.setattr(
+        settings,
+        '_cropped_file_size',
+        len(code)
+    )
+
+    foo, = names(code + code)
+    assert foo.line == 1
+
+    # It should just not crash if we are outside of the cropped range.
+    script = Script(code + code + 'Foo')
+    assert not script.goto_definitions()
+    assert 'Foo' in [c.name for c in script.completions()]