From db0e90763be0f65de1a03a270f10272b46184892 Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Fri, 10 Jul 2020 17:30:36 +0200 Subject: [PATCH] Start using pathlib.Path instead of all the os.path functions --- jedi/api/__init__.py | 23 +++++------ jedi/api/helpers.py | 5 ++- jedi/api/project.py | 59 ++++++++++++++--------------- jedi/api/refactoring/__init__.py | 22 ++++------- jedi/common.py | 12 ------ jedi/inference/__init__.py | 2 + jedi/inference/compiled/__init__.py | 2 +- jedi/inference/gradual/typeshed.py | 4 +- jedi/inference/helpers.py | 5 ++- jedi/inference/imports.py | 14 ++++--- jedi/inference/sys_path.py | 38 +++++++++---------- jedi/inference/value/module.py | 15 ++++---- 12 files changed, 97 insertions(+), 104 deletions(-) diff --git a/jedi/api/__init__.py b/jedi/api/__init__.py index 54631951..a2098bef 100644 --- a/jedi/api/__init__.py +++ b/jedi/api/__init__.py @@ -7,9 +7,9 @@ Alternatively, if you don't need a custom function and are happy with printing debug messages to stdout, simply call :func:`set_debug_function` without arguments. """ -import os import sys import warnings +from pathlib import Path import parso from parso.python import tree @@ -96,7 +96,7 @@ class Script(object): :type column: int :param path: The path of the file in the file system, or ``''`` if it hasn't been saved yet. - :type path: str or None + :type path: str or pathlib.Path or None :param sys_path: Deprecated, use the project parameter. :type sys_path: typing.List[str] :param Environment environment: Provide a predefined :ref:`Environment ` @@ -109,7 +109,10 @@ class Script(object): sys_path=None, environment=None, project=None, source=None): self._orig_path = path # An empty path (also empty string) should always result in no path. - self.path = os.path.abspath(path) if path else None + if isinstance(path, str): + path = Path(path) + + self.path = path.absolute() if path else None if line is not None: warnings.warn( @@ -139,9 +142,7 @@ class Script(object): if project is None: # Load the Python grammar of the current interpreter. - project = get_default_project( - os.path.dirname(self.path) if path else None - ) + project = get_default_project(self.path) # TODO deprecate and remove sys_path from the Script API. if sys_path is not None: project._sys_path = sys_path @@ -159,7 +160,7 @@ class Script(object): self._module_node, code = self._inference_state.parse_and_get_code( code=code, path=self.path, - use_latest_grammar=path and path.endswith('.pyi'), + use_latest_grammar=path and path.suffix == 'pyi', cache=False, # No disk cache, because the current script often changes. diff_cache=settings.fast_parser, cache_path=settings.cache_directory, @@ -191,7 +192,7 @@ class Script(object): file_io = None else: file_io = KnownContentFileIO(cast_path(self.path), self._code) - if self.path is not None and self.path.endswith('.pyi'): + if self.path is not None and self.path.suffix == 'pyi': # We are in a stub file. Try to load the stub properly. stub_module = load_proper_stub_module( self._inference_state, @@ -798,7 +799,7 @@ class Interpreter(Script): raise TypeError("The environment needs to be an InterpreterEnvironment subclass.") super().__init__(code, environment=environment, - project=Project(os.getcwd()), **kwds) + project=Project(Path.cwd()), **kwds) self.namespaces = namespaces self._inference_state.allow_descriptor_getattr = self._allow_descriptor_getattr_default @@ -806,7 +807,7 @@ class Interpreter(Script): def _get_module_context(self): tree_module_value = ModuleValue( self._inference_state, self._module_node, - file_io=KnownContentFileIO(self.path, self._code), + file_io=KnownContentFileIO(str(self.path), self._code), string_names=('__main__',), code_lines=self._code_lines, ) @@ -841,7 +842,7 @@ def preload_module(*modules): """ for m in modules: s = "import %s as x; x." % m - Script(s, path=None).complete(1, len(s)) + Script(s).complete(1, len(s)) def set_debug_function(func_cb=debug.print_to_stdout, warnings=True, diff --git a/jedi/api/helpers.py b/jedi/api/helpers.py index 4ef606e3..7d3677f7 100644 --- a/jedi/api/helpers.py +++ b/jedi/api/helpers.py @@ -44,7 +44,10 @@ def match(string, like_name, fuzzy=False): def sorted_definitions(defs): # Note: `or ''` below is required because `module_path` could be - return sorted(defs, key=lambda x: (x.module_path or '', x.line or 0, x.column or 0, x.name)) + return sorted(defs, key=lambda x: (str(x.module_path) or '', + x.line or 0, + x.column or 0, + x.name)) def get_on_completion_name(module_node, lines, position): diff --git a/jedi/api/project.py b/jedi/api/project.py index 0ba79e17..056a3873 100644 --- a/jedi/api/project.py +++ b/jedi/api/project.py @@ -7,9 +7,9 @@ flexibility to define sys paths and Python interpreters for a project, Projects can be saved to disk and loaded again, to allow project definitions to be used across repositories. """ -import os -import errno import json +from pathlib import Path +from itertools import chain from jedi import debug from jedi.api.environment import get_cached_default_environment, create_environment @@ -22,7 +22,6 @@ from jedi.inference.sys_path import discover_buildout_paths from jedi.inference.cache import inference_state_as_method_param_cache from jedi.inference.references import recurse_find_python_folders_and_files, search_in_file_ios from jedi.file_io import FolderIO -from jedi.common import traverse_parents _CONFIG_FOLDER = '.jedi' _CONTAINS_POTENTIAL_PROJECT = \ @@ -67,11 +66,11 @@ class Project(object): @staticmethod def _get_config_folder_path(base_path): - return os.path.join(base_path, _CONFIG_FOLDER) + return base_path.joinpath(_CONFIG_FOLDER) @staticmethod def _get_json_path(base_path): - return os.path.join(Project._get_config_folder_path(base_path), 'project.json') + return Project._get_config_folder_path(base_path).joinpath('project.json') @classmethod def load(cls, path): @@ -100,12 +99,7 @@ class Project(object): data.pop('_django', None) # TODO make django setting public? data = {k.lstrip('_'): v for k, v in data.items()} - # TODO when dropping Python 2 use pathlib.Path.mkdir(parents=True, exist_ok=True) - try: - os.makedirs(self._get_config_folder_path(self._path)) - except OSError as e: - if e.errno != errno.EEXIST: - raise + self._path.mkdir(parents=True, exist_ok=True) with open(self._get_json_path(self._path), 'w') as f: return json.dump((_SERIALIZER_VERSION, data), f) @@ -130,7 +124,9 @@ class Project(object): """ def py2_comp(path, environment_path=None, load_unsafe_extensions=False, sys_path=None, added_sys_path=(), smart_sys_path=True): - self._path = os.path.abspath(path) + if isinstance(path, str): + path = Path(path).absolute() + self._path = path self._environment_path = environment_path self._sys_path = sys_path @@ -174,23 +170,27 @@ class Project(object): sys_path = list(self._sys_path) if self._smart_sys_path: - prefixed.append(self._path) + prefixed.append(str(self._path)) if inference_state.script_path is not None: - suffixed += discover_buildout_paths(inference_state, inference_state.script_path) + suffixed += discover_buildout_paths( + inference_state, + inference_state.script_path + ) if add_parent_paths: # Collect directories in upward search by: # 1. Skipping directories with __init__.py # 2. Stopping immediately when above self._path traversed = [] - for parent_path in traverse_parents(inference_state.script_path): - if parent_path == self._path or not parent_path.startswith(self._path): + for parent_path in inference_state.script_path.parents: + if parent_path == self._path \ + or self._path not in parent_path.parents: break if not add_init_paths \ - and os.path.isfile(os.path.join(parent_path, "__init__.py")): + and parent_path.joinpath("__init__.py").is_file(): continue - traversed.append(parent_path) + traversed.append(str(parent_path)) # AFAIK some libraries have imports like `foo.foo.bar`, which # leads to the conclusion to by default prefer longer paths @@ -198,7 +198,7 @@ class Project(object): suffixed += reversed(traversed) if self._django: - prefixed.append(self._path) + prefixed.append(str(self._path)) path = prefixed + sys_path + suffixed return list(_remove_duplicates_from_path(path)) @@ -259,7 +259,7 @@ class Project(object): name = wanted_names[0] stub_folder_name = name + '-stubs' - ios = recurse_find_python_folders_and_files(FolderIO(self._path)) + ios = recurse_find_python_folders_and_files(FolderIO(str(self._path))) file_ios = [] # 1. Search for modules in the current project @@ -280,8 +280,7 @@ class Project(object): continue else: file_ios.append(file_io) - file_name = os.path.basename(file_io.path) - if file_name in (name + '.py', name + '.pyi'): + if Path(file_io.path).name in (name + '.py', name + '.pyi'): m = load_module_from_path(inference_state, file_io).as_context() else: continue @@ -318,7 +317,7 @@ class Project(object): p for p in self._get_sys_path(inference_state) # Exclude folders that are handled by recursing of the Python # folders. - if not p.startswith(self._path) + if not p.startswith(str(self._path)) ] names = list(iter_module_names(inference_state, empty_module_context, sys_path)) yield from search_in_module( @@ -337,7 +336,7 @@ class Project(object): def _is_potential_project(path): for name in _CONTAINS_POTENTIAL_PROJECT: - if os.path.exists(os.path.join(path, name)): + if path.joinpath(name).exists(): return True return False @@ -345,7 +344,7 @@ def _is_potential_project(path): def _is_django_path(directory): """ Detects the path of the very well known Django library (if used) """ try: - with open(os.path.join(directory, 'manage.py'), 'rb') as f: + with open(directory.joinpath('manage.py'), 'rb') as f: return b"DJANGO_SETTINGS_MODULE" in f.read() except (FileNotFoundError, IsADirectoryError, PermissionError): return False @@ -362,12 +361,12 @@ def get_default_project(path=None): ``requirements.txt`` and ``MANIFEST.in``. """ if path is None: - path = os.getcwd() + path = Path.cwd() - check = os.path.realpath(path) + check = path.absolute() probable_path = None first_no_init_file = None - for dir in traverse_parents(check, include_current=True): + for dir in chain([check], check.parents): try: return Project.load(dir) except (FileNotFoundError, IsADirectoryError, PermissionError): @@ -376,7 +375,7 @@ def get_default_project(path=None): continue if first_no_init_file is None: - if os.path.exists(os.path.join(dir, '__init__.py')): + if dir.joinpath('__init__.py').exists(): # In the case that a __init__.py exists, it's in 99% just a # Python package and the project sits at least one level above. continue @@ -398,7 +397,7 @@ def get_default_project(path=None): if first_no_init_file is not None: return Project(first_no_init_file) - curdir = path if os.path.isdir(path) else os.path.dirname(path) + curdir = path if path.is_dir() else path.parent return Project(curdir) diff --git a/jedi/api/refactoring/__init__.py b/jedi/api/refactoring/__init__.py index d0f7a7b8..e73beee4 100644 --- a/jedi/api/refactoring/__init__.py +++ b/jedi/api/refactoring/__init__.py @@ -1,6 +1,3 @@ -from os.path import dirname, basename, join, relpath -import os -import re import difflib from parso import split_lines @@ -43,11 +40,11 @@ class ChangedFile(object): if self._from_path is None: from_p = '' else: - from_p = relpath(self._from_path, project_path) + from_p = self._from_path.relative_to(project_path) if self._to_path is None: to_p = '' else: - to_p = relpath(self._to_path, project_path) + to_p = self._to_path.relative_to(project_path) diff = difflib.unified_diff( old_lines, new_lines, fromfile=from_p, @@ -115,7 +112,7 @@ class Refactoring(object): project_path = self._inference_state.project.path for from_, to in self.get_renames(): text += 'rename from %s\nrename to %s\n' \ - % (relpath(from_, project_path), relpath(to, project_path)) + % (from_.relative_to(project_path), to.relative_to(project_path)) return text + ''.join(f.get_diff() for f in self.get_changed_files().values()) @@ -127,17 +124,14 @@ class Refactoring(object): f.apply() for old, new in self.get_renames(): - os.rename(old, new) + old.rename(new) def _calculate_rename(path, new_name): - name = basename(path) - dir_ = dirname(path) - if name in ('__init__.py', '__init__.pyi'): - parent_dir = dirname(dir_) - return dir_, join(parent_dir, new_name) - ending = re.search(r'\.pyi?$', name).group(0) - return path, join(dir_, new_name + ending) + dir_ = path.parent + if path.name in ('__init__.py', '__init__.pyi'): + return dir_, dir_.parent.joinpath(new_name) + return path, dir_.joinpath(new_name + path.suffix) def rename(inference_state, definitions, new_name): diff --git a/jedi/common.py b/jedi/common.py index 75b69299..eb4b4996 100644 --- a/jedi/common.py +++ b/jedi/common.py @@ -1,18 +1,6 @@ -import os from contextlib import contextmanager -def traverse_parents(path, include_current=False): - if not include_current: - path = os.path.dirname(path) - - previous = None - while previous != path: - yield path - previous = path - path = os.path.dirname(path) - - @contextmanager def monkeypatch(obj, attribute_name, new_value): """ diff --git a/jedi/inference/__init__.py b/jedi/inference/__init__.py index 68e3a3da..68217365 100644 --- a/jedi/inference/__init__.py +++ b/jedi/inference/__init__.py @@ -178,6 +178,8 @@ class InferenceState(object): def parse_and_get_code(self, code=None, path=None, use_latest_grammar=False, file_io=None, **kwargs): + if path is not None: + path = str(path) if code is None: if file_io is None: file_io = FileIO(path) diff --git a/jedi/inference/compiled/__init__.py b/jedi/inference/compiled/__init__.py index e2c88340..faf5d373 100644 --- a/jedi/inference/compiled/__init__.py +++ b/jedi/inference/compiled/__init__.py @@ -44,7 +44,7 @@ def create_simple_object(inference_state, obj): Only allows creations of objects that are easily picklable across Python versions. """ - assert type(obj) in (int, float, str, bytes, slice, complex, bool), obj + assert type(obj) in (int, float, str, bytes, slice, complex, bool), repr(obj) compiled_value = create_from_access_path( inference_state, inference_state.compiled_subprocess.create_simple_object(obj) diff --git a/jedi/inference/gradual/typeshed.py b/jedi/inference/gradual/typeshed.py index 2f7dcf55..10681834 100644 --- a/jedi/inference/gradual/typeshed.py +++ b/jedi/inference/gradual/typeshed.py @@ -196,8 +196,8 @@ def _try_to_load_stub(inference_state, import_names, python_value_set, file_paths = [] if c.is_namespace(): file_paths = [os.path.join(p, '__init__.pyi') for p in c.py__path__()] - elif file_path is not None and file_path.endswith('.py'): - file_paths = [file_path + 'i'] + elif file_path is not None and file_path.suffix == '.py': + file_paths = [str(file_path) + 'i'] for file_path in file_paths: m = _try_to_load_stub_from_file( diff --git a/jedi/inference/helpers.py b/jedi/inference/helpers.py index 32650ba8..0e344c24 100644 --- a/jedi/inference/helpers.py +++ b/jedi/inference/helpers.py @@ -12,11 +12,12 @@ def is_stdlib_path(path): # Python standard library paths look like this: # /usr/lib/python3.9/... # TODO The implementation below is probably incorrect and not complete. - if 'dist-packages' in path or 'site-packages' in path: + parts = path.parts + if 'dist-packages' in parts or 'site-packages' in parts: return False base_path = os.path.join(sys.prefix, 'lib', 'python') - return bool(re.match(re.escape(base_path) + r'\d.\d', path)) + return bool(re.match(re.escape(base_path) + r'\d.\d', str(path))) def deep_ast_copy(obj): diff --git a/jedi/inference/imports.py b/jedi/inference/imports.py index 4fcb4c1d..8de3fe84 100644 --- a/jedi/inference/imports.py +++ b/jedi/inference/imports.py @@ -9,6 +9,7 @@ This module also supports import autocompletion, which means to complete statements like ``from datetim`` (cursor at the end would return ``datetime``). """ import os +from pathlib import Path from parso.python import tree from parso.tree import search_ancestor @@ -237,7 +238,10 @@ class Importer(object): # inference we want to show the user as much as possible. # See GH #1446. self._inference_state.get_sys_path(add_init_paths=not is_completion) - + sys_path.check_sys_path_modifications(self._module_context) + + [ + str(p) for p + in sys_path.check_sys_path_modifications(self._module_context) + ] ) def follow(self): @@ -467,19 +471,19 @@ def load_module_from_path(inference_state, file_io, import_names=None, is_packag here to ensure that a random path is still properly loaded into the Jedi module structure. """ - path = file_io.path + path = Path(file_io.path) if import_names is None: e_sys_path = inference_state.get_sys_path() import_names, is_package = sys_path.transform_path_to_dotted(e_sys_path, path) else: assert isinstance(is_package, bool) - is_stub = file_io.path.endswith('.pyi') + is_stub = path.suffix == '.pyi' if is_stub: folder_io = file_io.get_parent_folder() if folder_io.path.endswith('-stubs'): folder_io = FolderIO(folder_io.path[:-6]) - if file_io.path.endswith('__init__.pyi'): + if path.name == '__init__.pyi': python_file_io = folder_io.get_file_io('__init__.py') else: python_file_io = folder_io.get_file_io(import_names[-1] + '.py') @@ -510,7 +514,7 @@ def load_module_from_path(inference_state, file_io, import_names=None, is_packag def load_namespace_from_path(inference_state, folder_io): import_names, is_package = sys_path.transform_path_to_dotted( inference_state.get_sys_path(), - folder_io.path + Path(folder_io.path) ) from jedi.inference.value.namespace import ImplicitNamespaceValue return ImplicitNamespaceValue(inference_state, import_names, [folder_io.path]) diff --git a/jedi/inference/sys_path.py b/jedi/inference/sys_path.py index fd27bff0..12e0f166 100644 --- a/jedi/inference/sys_path.py +++ b/jedi/inference/sys_path.py @@ -1,11 +1,11 @@ import os import re +from pathlib import Path, PurePath from importlib.machinery import all_suffixes from jedi.inference.cache import inference_state_method_cache from jedi.inference.base_value import ContextualizedNode from jedi.inference.helpers import is_string, get_str_or_none -from jedi.common import traverse_parents from jedi.parser_utils import get_cached_code_lines from jedi.file_io import FileIO from jedi import settings @@ -14,8 +14,9 @@ from jedi import debug _BUILDOUT_PATH_INSERTION_LIMIT = 10 -def _abs_path(module_context, path): - if os.path.isabs(path): +def _abs_path(module_context, path: str): + path = PurePath(path) + if path.is_absolute(): return path module_path = module_context.py__file__() @@ -24,8 +25,8 @@ def _abs_path(module_context, path): # system. return None - base_dir = os.path.dirname(module_path) - return os.path.abspath(os.path.join(base_dir, path)) + base_dir = module_path.parent + return base_dir.joinpath(path).absolute() def _paths_from_assignment(module_context, expr_stmt): @@ -169,14 +170,14 @@ def _get_paths_from_buildout_script(inference_state, buildout_script_path): yield path -def _get_parent_dir_with_file(path, filename): - for parent in traverse_parents(path): - if os.path.isfile(os.path.join(parent, filename)): +def _get_parent_dir_with_file(path: Path, filename): + for parent in path.parents: + if parent.joinpath(filename).is_file(): return parent return None -def _get_buildout_script_paths(search_path): +def _get_buildout_script_paths(search_path: Path): """ if there is a 'buildout.cfg' file in one of the parent directories of the given module it will return a list of all files in the buildout bin @@ -188,13 +189,13 @@ def _get_buildout_script_paths(search_path): project_root = _get_parent_dir_with_file(search_path, 'buildout.cfg') if not project_root: return - bin_path = os.path.join(project_root, 'bin') - if not os.path.exists(bin_path): + bin_path = project_root.joinpath('bin') + if not bin_path.exists(): return for filename in os.listdir(bin_path): try: - filepath = os.path.join(bin_path, filename) + filepath = bin_path.joinpath(filename) with open(filepath, 'r') as f: firstline = f.readline() if firstline.startswith('#!') and 'python' in firstline: @@ -208,8 +209,8 @@ def _get_buildout_script_paths(search_path): def remove_python_path_suffix(path): for suffix in all_suffixes() + ['.pyi']: - if path.endswith(suffix): - path = path[:-len(suffix)] + if path.suffix == suffix: + path = path.with_name(path.stem) break return path @@ -232,16 +233,15 @@ def transform_path_to_dotted(sys_path, module_path): # means that if someone uses an ending like .vim for a Python file, .vim # will be part of the returned dotted part. - is_package = module_path.endswith(os.path.sep + '__init__') + is_package = module_path.name == '__init__' if is_package: - # -1 to remove the separator - module_path = module_path[:-len('__init__') - 1] + module_path = module_path.parent def iter_potential_solutions(): for p in sys_path: - if module_path.startswith(p): + if str(module_path).startswith(p): # Strip the trailing slash/backslash - rest = module_path[len(p):] + rest = str(module_path)[len(p):] # On Windows a path can also use a slash. if rest.startswith(os.path.sep) or rest.startswith('/'): # Remove a slash in cases it's still there. diff --git a/jedi/inference/value/module.py b/jedi/inference/value/module.py index be27eb17..2091d88b 100644 --- a/jedi/inference/value/module.py +++ b/jedi/inference/value/module.py @@ -1,4 +1,5 @@ import os +from pathlib import Path from jedi.inference.cache import inference_state_method_cache from jedi.inference.names import AbstractNameDefinition, ModuleName @@ -89,9 +90,9 @@ class ModuleMixin(SubModuleDictMixin): names = ['__package__', '__doc__', '__name__'] # All the additional module attributes are strings. dct = dict((n, _ModuleAttributeName(self, n)) for n in names) - file = self.py__file__() - if file is not None: - dct['__file__'] = _ModuleAttributeName(self, '__file__', file) + path = self.py__file__() + if path is not None: + dct['__file__'] = _ModuleAttributeName(self, '__file__', str(path)) return dct def iter_star_filters(self): @@ -147,13 +148,13 @@ class ModuleValue(ModuleMixin, TreeValue): if file_io is None: self._path = None else: - self._path = file_io.path + self._path = Path(file_io.path) self.string_names = string_names # Optional[Tuple[str, ...]] self.code_lines = code_lines self._is_package = is_package def is_stub(self): - if self._path is not None and self._path.endswith('.pyi'): + if self._path is not None and self._path.suffix == '.pyi': # Currently this is the way how we identify stubs when e.g. goto is # used in them. This could be changed if stubs would be identified # sooner and used as StubModuleValue. @@ -165,14 +166,14 @@ class ModuleValue(ModuleMixin, TreeValue): return None return '.'.join(self.string_names) - def py__file__(self): + def py__file__(self) -> Path: """ In contrast to Python's __file__ can be None. """ if self._path is None: return None - return os.path.abspath(self._path) + return self._path.absolute() def is_package(self): return self._is_package