Add a test to avoid encoding issues. Fixes #1003

jedi/api/__init__.py

@@ -92,14 +92,18 @@ class Script(object):
             with open(path, 'rb') as f:
                 source = f.read()
 
-        self._module_node, code = self.evaluator.parse_and_get_code(
+        # Load the Python grammar of the current interpreter.
-            code=self._source,
+        self._grammar = parso.load_grammar()
+        project = Project(sys_path=sys_path)
+        self._evaluator = Evaluator(self._grammar, project)
+        self._module_node, source = self._evaluator.parse_and_get_code(
+            code=source,
             path=self.path,
             cache=False,  # No disk cache, because the current script often changes.
             diff_cache=True,
             cache_path=settings.cache_directory
         )
-        self._code_lines = split_lines(code)
+        self._code_lines = split_lines(source)
         line = max(len(self._code_lines), 1) if line is None else line
         if not (0 < line <= len(self._code_lines)):
             raise ValueError('`line` parameter is not in a valid range.')
@@ -114,10 +118,6 @@ class Script(object):
         cache.clear_time_caches()
         debug.reset_time()
 
-        # Load the Python grammar of the current interpreter.
-        self._grammar = parso.load_grammar()
-        project = Project(sys_path=sys_path)
-        self._evaluator = Evaluator(self._grammar, project)
         project.add_script_path(self.path)
         debug.speed('init')
 

jedi/evaluate/__init__.py

@@ -104,7 +104,7 @@ class Evaluator(object):
         project.add_evaluator(self)
 
         self.reset_recursion_limitations()
-        self.allow_different_encoding = False
+        self.allow_different_encoding = True
 
         # Constants
         self.BUILTINS = compiled.get_special_object(self, 'BUILTINS')

jedi/evaluate/imports.py

@@ -369,8 +369,9 @@ class Importer(object):
             else:
                 module_path = get_init_path(module_path)
         elif module_file:
-            code = module_file.read()
             module_file.close()
+            with open(module_path, 'rb') as f:
+                code = f.read()
 
         if isinstance(module_path, ImplicitNSInfo):
             from jedi.evaluate.context.namespace import ImplicitNamespaceContext

test/conftest.py

@@ -123,5 +123,5 @@ class StaticAnalysisCase(object):
 
 @pytest.fixture()
 def cwd_tmpdir(monkeypatch, tmpdir):
-    with helpers.set_cwd(tmpdir.dirpath):
+    with helpers.set_cwd(tmpdir.strpath):
         yield tmpdir

test/test_api/test_unicode.py

@@ -64,3 +64,12 @@ def test_complete_at_zero():
 
     s = Script("", 1, 0).completions()
     assert len(s) > 0
+
+
+def test_wrong_encoding(cwd_tmpdir):
+    x = cwd_tmpdir.join('x.py')
+    # Use both latin-1 and utf-8 (a really broken file).
+    x.write_binary(u'foobar = 1\nä'.encode('latin-1') + 'ä'.encode())
+
+    c, = Script('import x; x.foo', sys_path=['.']).completions()
+    assert c.name == 'foobar'