Refactor to avoid having unicode decode errors by default

jedi/evaluate/__init__.py

@@ -66,6 +66,7 @@ import sys
 
 from parso.python import tree
 import parso
+from parso import python_bytes_to_unicode
 
 from jedi import debug
 from jedi import parser_utils
@@ -103,6 +104,7 @@ class Evaluator(object):
         project.add_evaluator(self)
 
         self.reset_recursion_limitations()
+        self.allow_different_encoding = False
 
         # Constants
         self.BUILTINS = compiled.get_special_object(self, 'BUILTINS')
@@ -357,3 +359,15 @@ class Evaluator(object):
                 node = node.parent
             scope_node = parent_scope(node)
         return from_scope_node(scope_node, is_nested=True, node_is_object=node_is_object)
+
+    def parse_and_get_code(self, code, path, **kwargs):
+        if self.allow_different_encoding:
+            if code is None:
+                with open('rb') as f:
+                    code = f.read()
+            code = python_bytes_to_unicode(code, errors='replace')
+
+        return self.grammar.parse(code=code, path=path, **kwargs), code
+
+    def parse(self, *args, **kwargs):
+        return self.parse_and_get_code(*args, **kwargs)[0]

jedi/evaluate/compiled/mixed.py

@@ -106,7 +106,7 @@ class MixedObjectFilter(compiled.CompiledObjectFilter):
 
 @evaluator_function_cache()
 def _load_module(evaluator, path, python_object):
-    module = evaluator.grammar.parse(
+    module = evaluator.parse(
         path=path,
         cache=True,
         diff_cache=True,

jedi/evaluate/imports.py

@@ -484,7 +484,7 @@ def _load_module(evaluator, path=None, code=None, sys_path=None, parent_module=N
     if path is not None and path.endswith(('.py', '.zip', '.egg')) \
             and dotted_path not in settings.auto_import_modules:
 
-        module_node = evaluator.grammar.parse(
+        module_node = evaluator.parse(
             code=code, path=path, cache=True, diff_cache=True,
             cache_path=settings.cache_directory)
 

jedi/evaluate/sys_path.py

@@ -204,7 +204,7 @@ def detect_additional_paths(evaluator, script_path):
 
 def _get_paths_from_buildout_script(evaluator, buildout_script_path):
     try:
-        module_node = evaluator.grammar.parse(
+        module_node = evaluator.parse(
             path=buildout_script_path,
             cache=True,
             cache_path=settings.cache_directory