Use unicode sys paths always

jedi/_compatibility.py

@@ -226,13 +226,26 @@ def u(string):
     have to cast back to a unicode (and we now that we always deal with valid
     unicode, because we check that in the beginning).
     """
-    if is_py3:
-        return str(string)
-
-    if not isinstance(string, unicode):
-        return unicode(str(string), 'UTF-8')
+    if isinstance(string, bytes):
+        return str(string, encoding='UTF-8')
     return string
 
+
+def cast_path(obj):
+    """
+    Take a bytes or str path and cast it to unicode.
+
+    Apparently it is perfectly fine to pass both byte and unicode objects into
+    the sys.path. This probably means that byte paths are normal at other
+    places as well.
+
+    Since this just really complicates everything and Python 2.7 will be EOL
+    soon anyway, just go with always strings.
+    """
+    return unicode(obj, encoding='utf-8', errors='replace') \
+           if isinstance(obj, bytes) else obj
+
+
 try:
     import builtins  # module name in python 3
 except ImportError:

jedi/evaluate/compiled/access.py

@@ -153,6 +153,15 @@ class AccessPath(object):
     def __init__(self, accesses):
         self.accesses = accesses
 
+    # Writing both of these methods here looks a bit ridiculous. However with
+    # the differences of Python 2/3 it's actually necessary, because we will
+    # otherwise have a accesses attribute that is bytes instead of unicode.
+    def __getstate__(self):
+        return self.accesses
+
+    def __setstate__(self, value):
+        self.accesses = value
+
 
 def create_access_path(evaluator, obj):
     access = create_access(evaluator, obj)

jedi/evaluate/compiled/subprocess/functions.py

@@ -1,12 +1,14 @@
 import sys
 import os
 import imp
+
+from jedi._compatibility import find_module, cast_path
 from jedi.evaluate.compiled import access
-from jedi._compatibility import find_module
+from jedi import parser_utils
 
 
 def get_sys_path():
-    return sys.path
+    return list(map(cast_path, sys.path))
 
 
 def load_module(evaluator, **kwargs):
@@ -51,7 +53,7 @@ def get_module_info(evaluator, sys_path=None, full_name=None, **kwargs):
         code = module_file.read()
         module_file.close()
 
-    return code, module_path, is_pkg
+    return code, cast_path(module_path), is_pkg
 
 
 def _get_init_path(directory_path):
@@ -64,3 +66,7 @@ def _get_init_path(directory_path):
         if os.path.exists(path):
             return path
     return None
+
+
+def safe_literal_eval(evaluator, value):
+    return parser_utils.safe_literal_eval(value)

jedi/evaluate/syntax_tree.py

@@ -173,7 +173,7 @@ def eval_atom(context, atom):
         )
 
     elif isinstance(atom, tree.Literal):
-        string = parser_utils.safe_literal_eval(atom.value)
+        string = context.evaluator.compiled_subprocess.safe_literal_eval(atom.value)
         return ContextSet(compiled.create_simple_object(context.evaluator, string))
     else:
         c = atom.children