Use scandir on py3.5+ for less disk access on filename completion

On Python 3.5+, we can make use of scandir that not only list the
content of the directory as an iterator but caches some infomations (for
example, `is_dir()`; this avoid extra stats call to the underlying
filesytem and can be – according to pep 471 –  2x to 20 time faster
especially on NFS filesystem where stats call is expensive.

From a quick this is the only place where scandir would make sens, as
most other places only require the name.

Fixes 1381
This commit is contained in:
Matthias Bussonnier
2019-08-12 17:56:29 -07:00
parent 005f69390c
commit f47211c129
2 changed files with 50 additions and 5 deletions

View File

@@ -27,6 +27,27 @@ is_py35 = is_py3 and sys.version_info[1] >= 5
py_version = int(str(sys.version_info[0]) + str(sys.version_info[1]))
if is_py35:
"""
A super-minimal shim around listdir that behave like
scandir for the information we need.
"""
class _DirEntry:
def __init__(self, name, basepath):
self.name = name
self.basepath = basepath
def is_dir(self):
path_for_name = os.path.join(self.basepath, self.name)
return os.path.isdir(path_for_name)
def scandir(dir):
return [_DirEntry(name, dir) for name in os.listdir(dir)]
else:
from os import scandir
class DummyFile(object):
def __init__(self, loader, string):
self.loader = loader

View File

@@ -1,12 +1,36 @@
import os
import sys
from jedi._compatibility import FileNotFoundError, force_unicode
from jedi._compatibility import FileNotFoundError, force_unicode, scandir
from jedi.evaluate.names import AbstractArbitraryName
from jedi.api import classes
from jedi.evaluate.helpers import get_str_or_none
from jedi.parser_utils import get_string_quote
if sys.version_info < (3,6) or True:
"""
A super-minimal shim around listdir that behave like
scandir for the information we need.
"""
class DirEntry:
def __init__(self, name, basepath):
self.name = name
self.basepath = basepath
def is_dir(self):
path_for_name = os.path.join(self.basepath, self.name)
return os.path.isdir(path_for_name)
def scandir(dir):
return [DirEntry(name, dir) for name in os.listdir(dir)]
else:
from os import scandir
def file_name_completions(evaluator, module_context, start_leaf, string,
like_name, call_signatures_callback, code_lines, position):
# First we want to find out what can actually be changed as a name.
@@ -32,13 +56,13 @@ def file_name_completions(evaluator, module_context, start_leaf, string,
string = to_be_added + string
base_path = os.path.join(evaluator.project._path, string)
try:
listed = os.listdir(base_path)
listed = scandir(base_path)
except FileNotFoundError:
return
for name in listed:
for entry in listed:
name = entry.name
if name.startswith(must_start_with):
path_for_name = os.path.join(base_path, name)
if is_in_os_path_join or not os.path.isdir(path_for_name):
if is_in_os_path_join or not entry.is_dir():
if start_leaf.type == 'string':
quote = get_string_quote(start_leaf)
else: