diff --git a/jedi/api/__init__.py b/jedi/api/__init__.py index de4fa3e6..1dae24b9 100644 --- a/jedi/api/__init__.py +++ b/jedi/api/__init__.py @@ -15,7 +15,7 @@ import sys import parso from parso.python import tree -from parso.utils import source_to_unicode, splitlines +from parso.utils import source_to_unicode, split_lines from jedi.parser_utils import get_executable_nodes, get_statement_of_position from jedi import debug @@ -109,8 +109,9 @@ class Script(object): with open(path, 'rb') as f: source = f.read() - self._source = source_to_unicode(source, encoding) - self._code_lines = splitlines(self._source) + # TODO do we really want that? + self._source = source_to_unicode(source, encoding, errors='replace') + self._code_lines = split_lines(self._source) line = max(len(self._code_lines), 1) if line is None else line if not (0 < line <= len(self._code_lines)): raise ValueError('`line` parameter is not in a valid range.') diff --git a/jedi/api/helpers.py b/jedi/api/helpers.py index 34f76daa..09659f56 100644 --- a/jedi/api/helpers.py +++ b/jedi/api/helpers.py @@ -7,7 +7,7 @@ from textwrap import dedent from parso.python.parser import Parser from parso.python import tree -from parso.utils import splitlines +from parso.utils import split_lines from jedi._compatibility import u from jedi.evaluate.helpers import evaluate_call_of_leaf @@ -53,7 +53,7 @@ class OnErrorLeaf(Exception): def _is_on_comment(leaf, position): - comment_lines = splitlines(leaf.prefix) + comment_lines = split_lines(leaf.prefix) difference = leaf.start_pos[0] - position[0] prefix_start_pos = leaf.get_start_pos_of_prefix() if difference == 0: diff --git a/jedi/evaluate/imports.py b/jedi/evaluate/imports.py index ebe61106..3fc713f0 100644 --- a/jedi/evaluate/imports.py +++ b/jedi/evaluate/imports.py @@ -510,7 +510,7 @@ def get_modules_containing_name(evaluator, modules, name): def check_fs(path): with open(path, 'rb') as f: - code = source_to_unicode(f.read()) + code = source_to_unicode(f.read(), errors='replace') if name in code: module_name = os.path.basename(path)[:-3] # Remove `.py`. module = _load_module(evaluator, path, code) diff --git a/jedi/evaluate/representation.py b/jedi/evaluate/representation.py index 80697806..68902604 100644 --- a/jedi/evaluate/representation.py +++ b/jedi/evaluate/representation.py @@ -548,7 +548,7 @@ class ModuleContext(use_metaclass(CachedMetaClass, context.TreeContext)): init_path = self.py__file__() if os.path.basename(init_path) == '__init__.py': with open(init_path, 'rb') as f: - content = source_to_unicode(f.read()) + content = source_to_unicode(f.read(), errors='replace') # these are strings that need to be used for namespace packages, # the first one is ``pkgutil``, the second ``pkg_resources``. options = ('declare_namespace(__name__)', 'extend_path(__path__') diff --git a/jedi/refactoring.py b/jedi/refactoring.py index e81b0040..51bdd683 100644 --- a/jedi/refactoring.py +++ b/jedi/refactoring.py @@ -15,7 +15,7 @@ following functions (sometimes bug-prone): import difflib from jedi import common -from parso.utils import source_to_unicode, splitlines +from parso.utils import source_to_unicode, split_lines from jedi.evaluate import helpers @@ -83,7 +83,7 @@ def _rename(names, replace_str): with open(current_path) as f: source = f.read() - new_lines = splitlines(source_to_unicode(source)) + new_lines = split_lines(source_to_unicode(source)) old_lines = new_lines[:] nr, indent = name.line, name.column @@ -101,7 +101,7 @@ def extract(script, new_name): :type source: str :return: list of changed lines/changed files """ - new_lines = splitlines(source_to_unicode(script.source)) + new_lines = split_lines(source_to_unicode(script.source)) old_lines = new_lines[:] user_stmt = script._parser.user_stmt() @@ -160,7 +160,7 @@ def inline(script): """ :type script: api.Script """ - new_lines = splitlines(source_to_unicode(script.source)) + new_lines = split_lines(source_to_unicode(script.source)) dct = {} diff --git a/jedi/utils.py b/jedi/utils.py index 655dc1ba..40cfca0d 100644 --- a/jedi/utils.py +++ b/jedi/utils.py @@ -11,7 +11,7 @@ import re import os import sys -from parso.utils import splitlines +from parso.utils import split_lines from jedi import Interpreter from jedi.api.helpers import get_on_completion_name @@ -86,7 +86,7 @@ def setup_readline(namespace_module=__main__): logging.debug("Start REPL completion: " + repr(text)) interpreter = Interpreter(text, [namespace_module.__dict__]) - lines = splitlines(text) + lines = split_lines(text) position = (len(lines), len(lines[-1])) name = get_on_completion_name( interpreter._get_module_node(),