move source_to_unicode to common

jedi/modules.py

@@ -18,10 +18,9 @@ from __future__ import with_statement
 import re
 import sys
 import os
-from ast import literal_eval
 
-from jedi._compatibility import unicode
 from jedi import cache
+from jedi.common import source_to_unicode
 from jedi.parser import tokenizer as tokenize
 from jedi.parser import fast
 from jedi import debug
@@ -225,30 +224,3 @@ class ModuleWithCursor(object):
 
     def get_position_line(self):
         return self.get_line(self.position[0])[:self.position[1]]
-def source_to_unicode(source, encoding=None):
-    def detect_encoding():
-        """ For the implementation of encoding definitions in Python, look at:
-        http://www.python.org/dev/peps/pep-0263/
-        http://docs.python.org/2/reference/lexical_analysis.html#encoding-\
-                                                                declarations
-        """
-        byte_mark = literal_eval(r"b'\xef\xbb\xbf'")
-        if source.startswith(byte_mark):
-            # UTF-8 byte-order mark
-            return 'utf-8'
-
-        first_two_lines = re.match(r'(?:[^\n]*\n){0,2}', str(source)).group(0)
-        possible_encoding = re.search(r"coding[=:]\s*([-\w.]+)",
-                                      first_two_lines)
-        if possible_encoding:
-            return possible_encoding.group(1)
-        else:
-            # the default if nothing else has been set -> PEP 263
-            return encoding if encoding is not None else 'iso-8859-1'
-
-    if isinstance(source, unicode):
-        # only cast str/bytes
-        return source
-
-    # cast to unicode by default
-    return unicode(source, detect_encoding(), 'replace')