Merge pull request #399 from jorgenschaefer/dev

Always load source files in binary mode.

AUTHORS.txt

@@ -22,5 +22,6 @@ Akinori Hattori (@hattya)
 srusskih (@srusskih)
 Steven Silvester (@blink1073)
 Colin Duquesnoy (@ColinDuquesnoy) <colin.duquesnoy@gmail.com>
+Jorgen Schaefer (@jorgenschaefer) <contact@jorgenschaefer.de>
 
 Note: (@user) means a github user name.

jedi/_compatibility.py

@@ -31,9 +31,7 @@ def find_module_py33(string, path=None):
             module_file = None
         else:
             module_path = loader.get_filename(string)
-            module_ext = os.path.splitext(module_path)[1]
+            module_file = open(module_path, 'rb')
-            mode = 'rb' if module_ext in ['.pyc', '.so', '.pyd'] else 'r'
-            module_file = open(module_path, mode)
     except AttributeError:
         # ExtensionLoader has not attribute get_filename, instead it has a
         # path attribute that we can use to retrieve the module path

jedi/evaluate/imports.py

@@ -331,8 +331,8 @@ class _Importer(object):
                         deeper_paths.append(new)
                 return follow_path(directories, deeper_paths)
 
-        with open(os.path.join(found_path, '__init__.py')) as f:
+        with open(os.path.join(found_path, '__init__.py'), 'rb') as f:
-            content = f.read()
+            content = common.source_to_unicode(f.read())
             # these are strings that need to be used for namespace packages,
             # the first one is ``pkgutil``, the second ``pkg_resources``.
             options = 'declare_namespace(__name__)', 'extend_path(__path__'
@@ -406,7 +406,7 @@ class _Importer(object):
             # is a directory module
             if is_package_directory:
                 path += '/__init__.py'
-                with open(path) as f:
+                with open(path, 'rb') as f:
                     source = f.read()
             else:
                 source = current_namespace[0].read()
@@ -454,7 +454,7 @@ def load_module(path=None, source=None, name=None):
     def load(source):
         if path is not None and path.endswith('.py'):
             if source is None:
-                with open(path) as f:
+                with open(path, 'rb') as f:
                     source = f.read()
         else:
             return compiled.load_module(path, name)
@@ -481,7 +481,7 @@ def get_modules_containing_name(mods, name):
                 return None
 
     def check_fs(path):
-        with open(path) as f:
+        with open(path, 'rb') as f:
             source = source_to_unicode(f.read())
             if name in source:
                 return load_module(path, source)

test/test_regression.py

@@ -4,6 +4,7 @@ found a good place in any other testing module.
 """
 
 import os
+import sys
 import textwrap
 
 from .helpers import TestCase, cwd_at
@@ -154,3 +155,19 @@ class TestRegression(TestCase):
             "    yield 1\n" \
             "abc()."
         assert Script(s).completions()
+
+
+def test_loading_unicode_files_with_bad_global_charset(monkeypatch, tmpdir):
+    dirname = str(tmpdir.mkdir('jedi-test'))
+    filename1 = os.path.join(dirname, 'test1.py')
+    filename2 = os.path.join(dirname, 'test2.py')
+    if sys.version_info < (3, 0):
+        data = "# coding: latin-1\nfoo = 'm\xf6p'\n"
+    else:
+        data = "# coding: latin-1\nfoo = 'm\xf6p'\n".encode("latin-1")
+
+    with open(filename1, "wb") as f:
+        f.write(data)
+    s = Script("from test1 import foo\nfoo.",
+               line=2, column=4, path=filename2)
+    s.complete()