From fcf214b5483b3f07b32b79ea4fee954de67235e4 Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Fri, 31 May 2019 23:25:41 +0200 Subject: [PATCH] Start using file io when opening random modules --- jedi/api/__init__.py | 5 ++- jedi/evaluate/context/module.py | 4 +-- jedi/evaluate/imports.py | 52 ++++++++++++++---------------- jedi/file_io.py | 35 ++++++++++++++------ test/test_evaluate/test_imports.py | 5 +-- 5 files changed, 59 insertions(+), 42 deletions(-) diff --git a/jedi/api/__init__.py b/jedi/api/__init__.py index bc2083bb..df5fa02d 100644 --- a/jedi/api/__init__.py +++ b/jedi/api/__init__.py @@ -164,7 +164,10 @@ class Script(object): names = import_names is_package = is_p - file_io = KnownContentFileIO(cast_path(self.path), self._code) + if self.path is None: + file_io = None + else: + file_io = KnownContentFileIO(cast_path(self.path), self._code) if self.path is not None and self.path.endswith('.pyi'): # We are in a stub file. Try to load the stub properly. stub_module = load_proper_stub_module( diff --git a/jedi/evaluate/context/module.py b/jedi/evaluate/context/module.py index 197c8efa..1bee7037 100644 --- a/jedi/evaluate/context/module.py +++ b/jedi/evaluate/context/module.py @@ -179,8 +179,8 @@ class ModuleContext(ModuleMixin, TreeContext): parent_context=None, tree_node=module_node ) - self._file_io = file_io - if self._file_io is None: + self.file_io = file_io + if file_io is None: self._path = None else: self._path = file_io.path diff --git a/jedi/evaluate/imports.py b/jedi/evaluate/imports.py index 5fec9415..ee603a92 100644 --- a/jedi/evaluate/imports.py +++ b/jedi/evaluate/imports.py @@ -16,12 +16,12 @@ import os from parso.python import tree from parso.tree import search_ancestor from parso import python_bytes_to_unicode -from jedi.file_io import KnownContentFileIO from jedi._compatibility import (FileNotFoundError, ImplicitNSInfo, force_unicode, unicode) from jedi import debug from jedi import settings +from jedi.file_io import KnownContentFileIO, FolderIO from jedi.parser_utils import get_cached_code_lines from jedi.evaluate import sys_path from jedi.evaluate import helpers @@ -485,13 +485,14 @@ def _load_builtin_module(evaluator, import_names=None, sys_path=None): return module -def _load_module_from_path(evaluator, path, base_names, code): +def _load_module_from_path(evaluator, file_io, base_names): """ This should pretty much only be used for get_modules_containing_name. It's here to ensure that a random path is still properly loaded into the Jedi module structure. """ e_sys_path = evaluator.get_sys_path() + path = file_io.path if base_names: module_name = os.path.basename(path) module_name = sys_path.remove_python_path_suffix(module_name) @@ -504,7 +505,7 @@ def _load_module_from_path(evaluator, path, base_names, code): import_names, is_package = sys_path.transform_path_to_dotted(e_sys_path, path) module = _load_python_module( - evaluator, KnownContentFileIO(path, code), + evaluator, file_io, sys_path=e_sys_path, import_names=import_names, is_package=is_package, @@ -517,37 +518,34 @@ def get_modules_containing_name(evaluator, modules, name): """ Search a name in the directories of modules. """ - def check_directory(path): - d = os.path.dirname(os.path.abspath(path)) - for file_name in os.listdir(d): - path = os.path.join(d, file_name) + def check_directory(folder_io): + for file_name in folder_io.list(): if file_name.endswith('.py'): - yield path + yield folder_io.get_file_io(file_name) - def check_fs(path, base_names): + def check_fs(file_io, base_names): try: - f = open(path, 'rb') + code = file_io.read() except FileNotFoundError: return None - with f: - code = python_bytes_to_unicode(f.read(), errors='replace') + code = python_bytes_to_unicode(code, errors='replace') if name not in code: return None - return _load_module_from_path(evaluator, path, base_names, code) + new_file_io = KnownContentFileIO(file_io.path, code) + return _load_module_from_path(evaluator, new_file_io, base_names) # skip non python modules used_mod_paths = set() - path_with_names_to_be_checked = [] + folders_with_names_to_be_checked = [] for m in modules: - try: - path = m.py__file__() - except AttributeError: - pass - else: - if path is not None: - if path not in used_mod_paths: - used_mod_paths.add(path) - path_with_names_to_be_checked.append((path, m.py__package__())) + if m.file_io is not None: + path = m.file_io.path + if path not in used_mod_paths: + used_mod_paths.add(path) + folders_with_names_to_be_checked.append(( + m.file_io.get_parent_folder(), + m.py__package__() + )) yield m if not settings.dynamic_params_for_other_modules: @@ -556,10 +554,10 @@ def get_modules_containing_name(evaluator, modules, name): for p in settings.additional_dynamic_modules: p = os.path.abspath(p) if p not in used_mod_paths: - path_with_names_to_be_checked.append((p, None)) + folders_with_names_to_be_checked.append((FolderIO(p), None)) - for p, base_names in path_with_names_to_be_checked: - for file_path in check_directory(p): - m = check_fs(file_path, base_names) + for folder_io, base_names in folders_with_names_to_be_checked: + for file_io in check_directory(folder_io): + m = check_fs(file_io, base_names) if m is not None and not isinstance(m, compiled.CompiledObject): yield m diff --git a/jedi/file_io.py b/jedi/file_io.py index b3c3a39d..37d958ea 100644 --- a/jedi/file_io.py +++ b/jedi/file_io.py @@ -3,13 +3,31 @@ import os from parso import file_io -class _ListDirMixin(object): - def listdir(self): - directory = self.path - return os.listdir(directory) +class AbstractFolderIO(object): + def __init__(self, path): + self.path = path + + def list(self): + raise NotImplementedError + + def get_file_io(self, name): + raise NotImplementedError -class ZipFileIO(file_io.KnownContentFileIO): +class FolderIO(AbstractFolderIO): + def list(self): + return os.listdir(self.path) + + def get_file_io(self, name): + return FileIO(os.path.join(self.path, name)) + + +class FileIOFolderMixin(object): + def get_parent_folder(self): + return FolderIO(os.path.dirname(self.path)) + + +class ZipFileIO(file_io.KnownContentFileIO, FileIOFolderMixin): """For .zip and .egg archives""" def __init__(self, path, code, zip_path): super(ZipFileIO, self).__init__(path, code) @@ -21,13 +39,10 @@ class ZipFileIO(file_io.KnownContentFileIO): except OSError: # Python 3 would probably only need FileNotFoundError return None - def listdir(self): - return [] - -class FileIO(_ListDirMixin, file_io.FileIO): +class FileIO(file_io.FileIO, FileIOFolderMixin): pass -class KnownContentFileIO(file_io.KnownContentFileIO): +class KnownContentFileIO(file_io.KnownContentFileIO, FileIOFolderMixin): pass diff --git a/test/test_evaluate/test_imports.py b/test/test_evaluate/test_imports.py index cebaa360..48bb5995 100644 --- a/test/test_evaluate/test_imports.py +++ b/test/test_evaluate/test_imports.py @@ -6,7 +6,7 @@ Tests". import os import pytest -from parso.file_io import FileIO +from jedi.file_io import FileIO, KnownContentFileIO from jedi._compatibility import find_module_py33, find_module from jedi.evaluate import compiled @@ -338,7 +338,8 @@ def test_get_modules_containing_name(evaluator, path, goal, is_package): ] ) def test_load_module_from_path(evaluator, path, base_names, is_package, names): - m = imports._load_module_from_path(evaluator, path, base_names, '') + file_io = KnownContentFileIO(path, '') + m = imports._load_module_from_path(evaluator, file_io, base_names) assert m.is_package == is_package assert m.string_names == names