diff --git a/jedi/api/__init__.py b/jedi/api/__init__.py index 871dc84b..ec81e952 100644 --- a/jedi/api/__init__.py +++ b/jedi/api/__init__.py @@ -16,7 +16,7 @@ import parso from parso.python import tree from parso import python_bytes_to_unicode, split_lines -from jedi.parser_utils import get_executable_nodes, get_statement_of_position +from jedi.parser_utils import get_executable_nodes from jedi import debug from jedi import settings from jedi import cache @@ -92,9 +92,14 @@ class Script(object): with open(path, 'rb') as f: source = f.read() - # TODO do we really want that? - self._source = python_bytes_to_unicode(source, encoding, errors='replace') - self._code_lines = split_lines(self._source) + self._module_node, code = self.evaluator.parse_and_get_code( + code=self._source, + path=self.path, + cache=False, # No disk cache, because the current script often changes. + diff_cache=True, + cache_path=settings.cache_directory + ) + self._code_lines = split_lines(code) line = max(len(self._code_lines), 1) if line is None else line if not (0 < line <= len(self._code_lines)): raise ValueError('`line` parameter is not in a valid range.') @@ -116,23 +121,9 @@ class Script(object): project.add_script_path(self.path) debug.speed('init') - @cache.memoize_method - def _get_module_node(self): - return self._grammar.parse( - code=self._source, - path=self.path, - cache=False, # No disk cache, because the current script often changes. - diff_cache=True, - cache_path=settings.cache_directory - ) - @cache.memoize_method def _get_module(self): - module = ModuleContext( - self._evaluator, - self._get_module_node(), - self.path - ) + module = ModuleContext(self._evaluator, self._module_node, self.path) if self.path is not None: name = dotted_path_in_sys_path(self._evaluator.project.sys_path, self.path) if name is not None: @@ -171,10 +162,9 @@ class Script(object): :rtype: list of :class:`classes.Definition` """ - module_node = self._get_module_node() - leaf = module_node.get_name_of_position(self._pos) + leaf = self._module_node.get_name_of_position(self._pos) if leaf is None: - leaf = module_node.get_leaf_for_position(self._pos) + leaf = self._module_node.get_leaf_for_position(self._pos) if leaf is None: return [] @@ -205,7 +195,7 @@ class Script(object): else: yield name - tree_name = self._get_module_node().get_name_of_position(self._pos) + tree_name = self._module_node.get_name_of_position(self._pos) if tree_name is None: return [] context = self._evaluator.create_context(self._get_module(), tree_name) @@ -236,7 +226,7 @@ class Script(object): :rtype: list of :class:`classes.Definition` """ - tree_name = self._get_module_node().get_name_of_position(self._pos) + tree_name = self._module_node.get_name_of_position(self._pos) if tree_name is None: # Must be syntax return [] @@ -263,7 +253,7 @@ class Script(object): :rtype: list of :class:`classes.CallSignature` """ call_signature_details = \ - helpers.get_call_signature_details(self._get_module_node(), self._pos) + helpers.get_call_signature_details(self._module_node, self._pos) if call_signature_details is None: return [] @@ -288,10 +278,9 @@ class Script(object): def _analysis(self): self._evaluator.is_analysis = True - module_node = self._get_module_node() - self._evaluator.analysis_modules = [module_node] + self._evaluator.analysis_modules = [self._module_node] try: - for node in get_executable_nodes(module_node): + for node in get_executable_nodes(self._module_node): context = self._get_module().create_context(node) if node.type in ('funcdef', 'classdef'): # Resolve the decorators. @@ -360,10 +349,9 @@ class Interpreter(Script): self.namespaces = namespaces def _get_module(self): - parser_module = super(Interpreter, self)._get_module_node() return interpreter.MixedModuleContext( self._evaluator, - parser_module, + self._module_node, self.namespaces, path=self.path ) @@ -399,7 +387,7 @@ def names(source=None, path=None, encoding='utf-8', all_scopes=False, module_context.create_context(name if name.parent.type == 'file_input' else name.parent), name ) - ) for name in get_module_names(script._get_module_node(), all_scopes) + ) for name in get_module_names(script._module_node, all_scopes) ] return sorted(filter(def_ref_filter, defs), key=lambda x: (x.line, x.column)) diff --git a/jedi/evaluate/__init__.py b/jedi/evaluate/__init__.py index 20461071..f8e5f39e 100644 --- a/jedi/evaluate/__init__.py +++ b/jedi/evaluate/__init__.py @@ -66,6 +66,7 @@ import sys from parso.python import tree import parso +from parso import python_bytes_to_unicode from jedi import debug from jedi import parser_utils @@ -103,6 +104,7 @@ class Evaluator(object): project.add_evaluator(self) self.reset_recursion_limitations() + self.allow_different_encoding = False # Constants self.BUILTINS = compiled.get_special_object(self, 'BUILTINS') @@ -357,3 +359,15 @@ class Evaluator(object): node = node.parent scope_node = parent_scope(node) return from_scope_node(scope_node, is_nested=True, node_is_object=node_is_object) + + def parse_and_get_code(self, code, path, **kwargs): + if self.allow_different_encoding: + if code is None: + with open('rb') as f: + code = f.read() + code = python_bytes_to_unicode(code, errors='replace') + + return self.grammar.parse(code=code, path=path, **kwargs), code + + def parse(self, *args, **kwargs): + return self.parse_and_get_code(*args, **kwargs)[0] diff --git a/jedi/evaluate/compiled/mixed.py b/jedi/evaluate/compiled/mixed.py index 462ac7d5..6cc0d847 100644 --- a/jedi/evaluate/compiled/mixed.py +++ b/jedi/evaluate/compiled/mixed.py @@ -106,7 +106,7 @@ class MixedObjectFilter(compiled.CompiledObjectFilter): @evaluator_function_cache() def _load_module(evaluator, path, python_object): - module = evaluator.grammar.parse( + module = evaluator.parse( path=path, cache=True, diff_cache=True, diff --git a/jedi/evaluate/imports.py b/jedi/evaluate/imports.py index 89dd833f..8cc9b5b6 100644 --- a/jedi/evaluate/imports.py +++ b/jedi/evaluate/imports.py @@ -484,7 +484,7 @@ def _load_module(evaluator, path=None, code=None, sys_path=None, parent_module=N if path is not None and path.endswith(('.py', '.zip', '.egg')) \ and dotted_path not in settings.auto_import_modules: - module_node = evaluator.grammar.parse( + module_node = evaluator.parse( code=code, path=path, cache=True, diff_cache=True, cache_path=settings.cache_directory) diff --git a/jedi/evaluate/sys_path.py b/jedi/evaluate/sys_path.py index 82e5e9df..0a8485f2 100644 --- a/jedi/evaluate/sys_path.py +++ b/jedi/evaluate/sys_path.py @@ -204,7 +204,7 @@ def detect_additional_paths(evaluator, script_path): def _get_paths_from_buildout_script(evaluator, buildout_script_path): try: - module_node = evaluator.grammar.parse( + module_node = evaluator.parse( path=buildout_script_path, cache=True, cache_path=settings.cache_directory