Refactor to avoid having unicode decode errors by default

jedi/api/__init__.py

@@ -16,7 +16,7 @@ import parso
 from parso.python import tree
 from parso import python_bytes_to_unicode, split_lines
 
-from jedi.parser_utils import get_executable_nodes, get_statement_of_position
+from jedi.parser_utils import get_executable_nodes
 from jedi import debug
 from jedi import settings
 from jedi import cache
@@ -92,9 +92,14 @@ class Script(object):
             with open(path, 'rb') as f:
                 source = f.read()
 
-        # TODO do we really want that?
-        self._source = python_bytes_to_unicode(source, encoding, errors='replace')
-        self._code_lines = split_lines(self._source)
+        self._module_node, code = self.evaluator.parse_and_get_code(
+            code=self._source,
+            path=self.path,
+            cache=False,  # No disk cache, because the current script often changes.
+            diff_cache=True,
+            cache_path=settings.cache_directory
+        )
+        self._code_lines = split_lines(code)
         line = max(len(self._code_lines), 1) if line is None else line
         if not (0 < line <= len(self._code_lines)):
             raise ValueError('`line` parameter is not in a valid range.')
@@ -116,23 +121,9 @@ class Script(object):
         project.add_script_path(self.path)
         debug.speed('init')
 
-    @cache.memoize_method
-    def _get_module_node(self):
-        return self._grammar.parse(
-            code=self._source,
-            path=self.path,
-            cache=False,  # No disk cache, because the current script often changes.
-            diff_cache=True,
-            cache_path=settings.cache_directory
-        )
-
     @cache.memoize_method
     def _get_module(self):
-        module = ModuleContext(
-            self._evaluator,
-            self._get_module_node(),
-            self.path
-        )
+        module = ModuleContext(self._evaluator, self._module_node, self.path)
         if self.path is not None:
             name = dotted_path_in_sys_path(self._evaluator.project.sys_path, self.path)
             if name is not None:
@@ -171,10 +162,9 @@ class Script(object):
 
         :rtype: list of :class:`classes.Definition`
         """
-        module_node = self._get_module_node()
-        leaf = module_node.get_name_of_position(self._pos)
+        leaf = self._module_node.get_name_of_position(self._pos)
         if leaf is None:
-            leaf = module_node.get_leaf_for_position(self._pos)
+            leaf = self._module_node.get_leaf_for_position(self._pos)
             if leaf is None:
                 return []
 
@@ -205,7 +195,7 @@ class Script(object):
                 else:
                     yield name
 
-        tree_name = self._get_module_node().get_name_of_position(self._pos)
+        tree_name = self._module_node.get_name_of_position(self._pos)
         if tree_name is None:
             return []
         context = self._evaluator.create_context(self._get_module(), tree_name)
@@ -236,7 +226,7 @@ class Script(object):
 
         :rtype: list of :class:`classes.Definition`
         """
-        tree_name = self._get_module_node().get_name_of_position(self._pos)
+        tree_name = self._module_node.get_name_of_position(self._pos)
         if tree_name is None:
             # Must be syntax
             return []
@@ -263,7 +253,7 @@ class Script(object):
         :rtype: list of :class:`classes.CallSignature`
         """
         call_signature_details = \
-            helpers.get_call_signature_details(self._get_module_node(), self._pos)
+            helpers.get_call_signature_details(self._module_node, self._pos)
         if call_signature_details is None:
             return []
 
@@ -288,10 +278,9 @@ class Script(object):
 
     def _analysis(self):
         self._evaluator.is_analysis = True
-        module_node = self._get_module_node()
-        self._evaluator.analysis_modules = [module_node]
+        self._evaluator.analysis_modules = [self._module_node]
         try:
-            for node in get_executable_nodes(module_node):
+            for node in get_executable_nodes(self._module_node):
                 context = self._get_module().create_context(node)
                 if node.type in ('funcdef', 'classdef'):
                     # Resolve the decorators.
@@ -360,10 +349,9 @@ class Interpreter(Script):
         self.namespaces = namespaces
 
     def _get_module(self):
-        parser_module = super(Interpreter, self)._get_module_node()
         return interpreter.MixedModuleContext(
             self._evaluator,
-            parser_module,
+            self._module_node,
             self.namespaces,
             path=self.path
         )
@@ -399,7 +387,7 @@ def names(source=None, path=None, encoding='utf-8', all_scopes=False,
                 module_context.create_context(name if name.parent.type == 'file_input' else name.parent),
                 name
             )
-        ) for name in get_module_names(script._get_module_node(), all_scopes)
+        ) for name in get_module_names(script._module_node, all_scopes)
     ]
     return sorted(filter(def_ref_filter, defs), key=lambda x: (x.line, x.column))
 

jedi/evaluate/__init__.py

@@ -66,6 +66,7 @@ import sys
 
 from parso.python import tree
 import parso
+from parso import python_bytes_to_unicode
 
 from jedi import debug
 from jedi import parser_utils
@@ -103,6 +104,7 @@ class Evaluator(object):
         project.add_evaluator(self)
 
         self.reset_recursion_limitations()
+        self.allow_different_encoding = False
 
         # Constants
         self.BUILTINS = compiled.get_special_object(self, 'BUILTINS')
@@ -357,3 +359,15 @@ class Evaluator(object):
                 node = node.parent
             scope_node = parent_scope(node)
         return from_scope_node(scope_node, is_nested=True, node_is_object=node_is_object)
+
+    def parse_and_get_code(self, code, path, **kwargs):
+        if self.allow_different_encoding:
+            if code is None:
+                with open('rb') as f:
+                    code = f.read()
+            code = python_bytes_to_unicode(code, errors='replace')
+
+        return self.grammar.parse(code=code, path=path, **kwargs), code
+
+    def parse(self, *args, **kwargs):
+        return self.parse_and_get_code(*args, **kwargs)[0]

jedi/evaluate/compiled/mixed.py

@@ -106,7 +106,7 @@ class MixedObjectFilter(compiled.CompiledObjectFilter):
 
 @evaluator_function_cache()
 def _load_module(evaluator, path, python_object):
-    module = evaluator.grammar.parse(
+    module = evaluator.parse(
         path=path,
         cache=True,
         diff_cache=True,

jedi/evaluate/imports.py

@@ -484,7 +484,7 @@ def _load_module(evaluator, path=None, code=None, sys_path=None, parent_module=N
     if path is not None and path.endswith(('.py', '.zip', '.egg')) \
             and dotted_path not in settings.auto_import_modules:
 
-        module_node = evaluator.grammar.parse(
+        module_node = evaluator.parse(
             code=code, path=path, cache=True, diff_cache=True,
             cache_path=settings.cache_directory)
 

jedi/evaluate/sys_path.py

@@ -204,7 +204,7 @@ def detect_additional_paths(evaluator, script_path):
 
 def _get_paths_from_buildout_script(evaluator, buildout_script_path):
     try:
-        module_node = evaluator.grammar.parse(
+        module_node = evaluator.parse(
             path=buildout_script_path,
             cache=True,
             cache_path=settings.cache_directory