switched completely to unicode (in python2), fixes #52

jedi/_compatibility.py

@@ -58,8 +58,7 @@ else:
 try:
     unicode = unicode
 except NameError:
-    def unicode(s):
-        return s.decode("utf-8")
+    unicode = str
 
 # exec function
 if is_py3k:
@@ -78,9 +77,9 @@ else:
 
 # BytesIO (Python 2.5 has no io module)
 try:
-    from cStringIO import StringIO as BytesIO
+    from StringIO import StringIO as BytesIO
 except ImportError:
-    from io import BytesIO
+    from io import BytesIO as BytesIO
 
 # hasattr function used because python
 if is_py3k:

jedi/api.py

@@ -14,7 +14,7 @@ import keywords
 import helpers
 import builtin
 
-from _compatibility import next
+from _compatibility import next, unicode
 
 
 class NotFoundError(Exception):
@@ -201,6 +201,7 @@ class Script(object):
     :type source_path: string or None
     """
     def __init__(self, source, line, column, source_path):
+        source = unicode(source)
         self.pos = line, column
         self.module = modules.ModuleWithCursor(source_path, source=source,
                                                             position=self.pos)

jedi/evaluate.py

@@ -14,7 +14,8 @@ TODO nonlocal statement, needed or can be ignored? (py3k)
 
 TODO __ instance attributes should not be visible outside of the class.
 """
-from _compatibility import next, property, hasattr, is_py3k, use_metaclass
+from _compatibility import next, property, hasattr, is_py3k, use_metaclass, \
+                            unicode
 
 import sys
 import itertools
@@ -1407,7 +1408,7 @@ def follow_call_list(call_list):
                                             dynamic.ArrayInstance)):
                         result.append(call)
                     # The string tokens are just operations (+, -, etc.)
-                    elif not isinstance(call, str):
+                    elif not isinstance(call, (str, unicode)):
                         if str(call.name) == 'if':
                             # Ternary operators.
                             while True:

jedi/keywords.py

@@ -59,6 +59,9 @@ def imitate_pydoc(string):
     It's not possible to get the pydoc's without starting the annoying pager
     stuff.
     """
+    # str needed because of possible unicode stuff in py2k (pydoc doesn't work
+    # with unicode strings)
+    string = str(string)
     h = pydoc.help
     try:
         # try to access symbols

test/run.py

@@ -64,7 +64,7 @@ def run_goto_test(script, correct, line_nr):
     Return if the test was a fail or not, with 1 for fail and 0 for success.
     """
     result = script.goto()
-    comp_str = str(sorted(r.description for r in result))
+    comp_str = str(sorted(str(r.description) for r in result))
     if comp_str != correct:
         print('Solution @%s not right, received %s, wanted %s'\
                     % (line_nr - 1, comp_str, correct))
@@ -142,7 +142,7 @@ def run_test(source, f_name, lines_to_execute):
     correct = None
     test_type = None
     start = None
-    for line_nr, line in enumerate(BytesIO(source.encode())):
+    for line_nr, line in enumerate(BytesIO(source)):
         line = unicode(line)
         line_nr += 1
         if correct:
@@ -212,7 +212,8 @@ def test_dir(completion_test_dir, thirdparty=False):
 
             path = os.path.join(completion_test_dir, f_name)
             f = open(path)
-            num_tests, fails = run_test(f.read(), f_name, lines_to_execute)
+            num_tests, fails = run_test(unicode(f.read()), f_name,
+                                                            lines_to_execute)
 
             s = 'run %s tests with %s fails (%s)' % (num_tests, fails, f_name)
             tests_fail += fails