diff --git a/parso/_compatibility.py b/parso/_compatibility.py index 43e1d86..4c966d6 100644 --- a/parso/_compatibility.py +++ b/parso/_compatibility.py @@ -50,6 +50,12 @@ try: except NameError: # Python 2.7 (both IOError + OSError) FileNotFoundError = EnvironmentError +try: + # Python 3.3+ + PermissionError = PermissionError +except NameError: + # Python 2.7 (both IOError + OSError) + PermissionError = EnvironmentError def utf8_repr(func): @@ -69,6 +75,7 @@ def utf8_repr(func): else: return wrapper + if sys.version_info < (3, 5): """ A super-minimal shim around listdir that behave like diff --git a/parso/cache.py b/parso/cache.py index 5e43a1a..8644423 100644 --- a/parso/cache.py +++ b/parso/cache.py @@ -7,13 +7,14 @@ import shutil import platform import errno import logging +import warnings try: import cPickle as pickle except: import pickle -from parso._compatibility import FileNotFoundError, scandir +from parso._compatibility import FileNotFoundError, PermissionError, scandir from parso.file_io import FileIO LOG = logging.getLogger(__name__) @@ -182,7 +183,7 @@ def _set_cache_item(hashed_grammar, path, module_cache_item): parser_cache.setdefault(hashed_grammar, {})[path] = module_cache_item -def save_module(hashed_grammar, file_io, module, lines, pickling=True, cache_path=None): +def try_to_save_module(hashed_grammar, file_io, module, lines, pickling=True, cache_path=None): path = file_io.path try: p_time = None if path is None else file_io.get_last_modified() @@ -193,8 +194,18 @@ def save_module(hashed_grammar, file_io, module, lines, pickling=True, cache_pat item = _NodeCacheItem(module, lines, p_time) _set_cache_item(hashed_grammar, path, item) if pickling and path is not None: - _save_to_file_system(hashed_grammar, path, item, cache_path=cache_path) - _remove_cache_and_update_lock(cache_path = cache_path) + try: + _save_to_file_system(hashed_grammar, path, item, cache_path=cache_path) + except PermissionError: + # It's not really a big issue if the cache cannot be saved to the + # file system. It's still in RAM in that case. However we should + # still warn the user that this is happening. + warnings.warn( + 'Tried to save a file to %s, but got permission denied.', + Warning + ) + else: + _remove_cache_and_update_lock(cache_path=cache_path) def _save_to_file_system(hashed_grammar, path, item, cache_path=None): diff --git a/parso/grammar.py b/parso/grammar.py index 0742d1c..6233005 100644 --- a/parso/grammar.py +++ b/parso/grammar.py @@ -7,7 +7,7 @@ from parso.utils import split_lines, python_bytes_to_unicode, parse_version_stri from parso.python.diff import DiffParser from parso.python.tokenize import tokenize_lines, tokenize from parso.python.token import PythonTokenTypes -from parso.cache import parser_cache, load_module, save_module +from parso.cache import parser_cache, load_module, try_to_save_module from parso.parser import BaseParser from parso.python.parser import Parser as PythonParser from parso.python.errors import ErrorFinderConfig @@ -132,7 +132,7 @@ class Grammar(object): old_lines=old_lines, new_lines=lines ) - save_module(self._hashed, file_io, new_node, lines, + try_to_save_module(self._hashed, file_io, new_node, lines, # Never pickle in pypy, it's slow as hell. pickling=cache and not is_pypy, cache_path=cache_path) @@ -148,7 +148,7 @@ class Grammar(object): root_node = p.parse(tokens=tokens) if cache or diff_cache: - save_module(self._hashed, file_io, root_node, lines, + try_to_save_module(self._hashed, file_io, root_node, lines, # Never pickle in pypy, it's slow as hell. pickling=cache and not is_pypy, cache_path=cache_path) diff --git a/test/test_cache.py b/test/test_cache.py index f9528bb..e1a0a9e 100644 --- a/test/test_cache.py +++ b/test/test_cache.py @@ -12,14 +12,19 @@ from parso.cache import (_CACHED_FILE_MAXIMUM_SURVIVAL, _VERSION_TAG, _get_cache_clear_lock, _get_hashed_path, _load_from_file_system, _NodeCacheItem, _remove_cache_and_update_lock, _save_to_file_system, - clear_inactive_cache, load_module, parser_cache, - save_module) -from parso._compatibility import is_pypy + load_module, parser_cache, try_to_save_module) +from parso._compatibility import is_pypy, PermissionError from parso import load_grammar from parso import cache from parso import file_io from parso import parse +skip_pypy = pytest.mark.skipif( + is_pypy, + reason="pickling in pypy is slow, since we don't pickle," + "we never go into path of auto-collecting garbage" +) + @pytest.fixture() def isolated_parso_cache(monkeypatch, tmpdir): @@ -30,6 +35,7 @@ def isolated_parso_cache(monkeypatch, tmpdir): monkeypatch.setattr(cache, '_get_default_cache_path', lambda *args, **kwargs: cache_path) return cache_path + def test_modulepickling_change_cache_dir(tmpdir): """ ParserPickling should not save old cache when cache_directory is changed. @@ -85,7 +91,7 @@ def test_modulepickling_simulate_deleted_cache(tmpdir): pass io = file_io.FileIO(path) - save_module(grammar._hashed, io, module, lines=[]) + try_to_save_module(grammar._hashed, io, module, lines=[]) assert load_module(grammar._hashed, io) == module os.unlink(_get_hashed_path(grammar._hashed, path)) @@ -144,11 +150,8 @@ def test_cache_last_used_update(diff_cache, use_file_io): node_cache_item = next(iter(parser_cache.values()))[p] assert now < node_cache_item.last_used < time.time() -@pytest.mark.skipif( - is_pypy, - reason="pickling in pypy is slow, since we don't pickle," - "we never go into path of auto-collecting garbage" -) + +@skip_pypy def test_inactive_cache(tmpdir, isolated_parso_cache): parser_cache.clear() test_subjects = "abcdef" @@ -159,12 +162,12 @@ def test_inactive_cache(tmpdir, isolated_parso_cache): paths = os.listdir(raw_cache_path) a_while_ago = time.time() - _CACHED_FILE_MAXIMUM_SURVIVAL old_paths = set() - for path in paths[:len(test_subjects) // 2]: # make certain number of paths old + for path in paths[:len(test_subjects) // 2]: # make certain number of paths old os.utime(os.path.join(raw_cache_path, path), (a_while_ago, a_while_ago)) old_paths.add(path) # nothing should be cleared while the lock is on assert os.path.exists(_get_cache_clear_lock().path) - _remove_cache_and_update_lock() # it shouldn't clear anything + _remove_cache_and_update_lock() # it shouldn't clear anything assert len(os.listdir(raw_cache_path)) == len(test_subjects) assert old_paths.issubset(os.listdir(raw_cache_path)) @@ -172,3 +175,17 @@ def test_inactive_cache(tmpdir, isolated_parso_cache): _remove_cache_and_update_lock() assert len(os.listdir(raw_cache_path)) == len(test_subjects) // 2 assert not old_paths.intersection(os.listdir(raw_cache_path)) + + +@skip_pypy +def test_permission_error(monkeypatch): + def save(*args, **kwargs): + was_called[0] = True # Python 2... Use nonlocal instead + raise PermissionError + + was_called = [False] + + monkeypatch.setattr(cache, '_save_to_file_system', save) + with pytest.warns(Warning): + parse(path=__file__, cache=True, diff_cache=True) + assert was_called[0]