Merge pull request #136 from davidhalter/permission_errors

Ignore permission errors when saving to cache

parso/_compatibility.py

@@ -50,6 +50,12 @@ try:
 except NameError:
     # Python 2.7 (both IOError + OSError)
     FileNotFoundError = EnvironmentError
+try:
+    # Python 3.3+
+    PermissionError = PermissionError
+except NameError:
+    # Python 2.7 (both IOError + OSError)
+    PermissionError = EnvironmentError
 
 
 def utf8_repr(func):
@@ -69,6 +75,7 @@ def utf8_repr(func):
     else:
         return wrapper
 
+
 if sys.version_info < (3, 5):
     """
     A super-minimal shim around listdir that behave like

parso/cache.py

@@ -7,13 +7,14 @@ import shutil
 import platform
 import errno
 import logging
+import warnings
 
 try:
     import cPickle as pickle
 except:
     import pickle
 
-from parso._compatibility import FileNotFoundError, scandir
+from parso._compatibility import FileNotFoundError, PermissionError, scandir
 from parso.file_io import FileIO
 
 LOG = logging.getLogger(__name__)
@@ -182,7 +183,7 @@ def _set_cache_item(hashed_grammar, path, module_cache_item):
     parser_cache.setdefault(hashed_grammar, {})[path] = module_cache_item
 
 
-def save_module(hashed_grammar, file_io, module, lines, pickling=True, cache_path=None):
+def try_to_save_module(hashed_grammar, file_io, module, lines, pickling=True, cache_path=None):
     path = file_io.path
     try:
         p_time = None if path is None else file_io.get_last_modified()
@@ -193,8 +194,18 @@ def save_module(hashed_grammar, file_io, module, lines, pickling=True, cache_pat
     item = _NodeCacheItem(module, lines, p_time)
     _set_cache_item(hashed_grammar, path, item)
     if pickling and path is not None:
-        _save_to_file_system(hashed_grammar, path, item, cache_path=cache_path)
+        try:
-        _remove_cache_and_update_lock(cache_path = cache_path)
+            _save_to_file_system(hashed_grammar, path, item, cache_path=cache_path)
+        except PermissionError:
+            # It's not really a big issue if the cache cannot be saved to the
+            # file system. It's still in RAM in that case. However we should
+            # still warn the user that this is happening.
+            warnings.warn(
+                'Tried to save a file to %s, but got permission denied.',
+                Warning
+            )
+        else:
+            _remove_cache_and_update_lock(cache_path=cache_path)
 
 
 def _save_to_file_system(hashed_grammar, path, item, cache_path=None):

parso/grammar.py

@@ -7,7 +7,7 @@ from parso.utils import split_lines, python_bytes_to_unicode, parse_version_stri
 from parso.python.diff import DiffParser
 from parso.python.tokenize import tokenize_lines, tokenize
 from parso.python.token import PythonTokenTypes
-from parso.cache import parser_cache, load_module, save_module
+from parso.cache import parser_cache, load_module, try_to_save_module
 from parso.parser import BaseParser
 from parso.python.parser import Parser as PythonParser
 from parso.python.errors import ErrorFinderConfig
@@ -132,7 +132,7 @@ class Grammar(object):
                     old_lines=old_lines,
                     new_lines=lines
                 )
-                save_module(self._hashed, file_io, new_node, lines,
+                try_to_save_module(self._hashed, file_io, new_node, lines,
                             # Never pickle in pypy, it's slow as hell.
                             pickling=cache and not is_pypy,
                             cache_path=cache_path)
@@ -148,7 +148,7 @@ class Grammar(object):
         root_node = p.parse(tokens=tokens)
 
         if cache or diff_cache:
-            save_module(self._hashed, file_io, root_node, lines,
+            try_to_save_module(self._hashed, file_io, root_node, lines,
                         # Never pickle in pypy, it's slow as hell.
                         pickling=cache and not is_pypy,
                         cache_path=cache_path)

test/test_cache.py

@@ -12,14 +12,19 @@ from parso.cache import (_CACHED_FILE_MAXIMUM_SURVIVAL, _VERSION_TAG,
                          _get_cache_clear_lock, _get_hashed_path,
                          _load_from_file_system, _NodeCacheItem,
                          _remove_cache_and_update_lock, _save_to_file_system,
-                         clear_inactive_cache, load_module, parser_cache,
+                         load_module, parser_cache, try_to_save_module)
-                         save_module)
+from parso._compatibility import is_pypy, PermissionError
-from parso._compatibility import is_pypy
 from parso import load_grammar
 from parso import cache
 from parso import file_io
 from parso import parse
 
+skip_pypy = pytest.mark.skipif(
+    is_pypy,
+    reason="pickling in pypy is slow, since we don't pickle,"
+           "we never go into path of auto-collecting garbage"
+)
+
 
 @pytest.fixture()
 def isolated_parso_cache(monkeypatch, tmpdir):
@@ -30,6 +35,7 @@ def isolated_parso_cache(monkeypatch, tmpdir):
     monkeypatch.setattr(cache, '_get_default_cache_path', lambda *args, **kwargs: cache_path)
     return cache_path
 
+
 def test_modulepickling_change_cache_dir(tmpdir):
     """
     ParserPickling should not save old cache when cache_directory is changed.
@@ -85,7 +91,7 @@ def test_modulepickling_simulate_deleted_cache(tmpdir):
         pass
     io = file_io.FileIO(path)
 
-    save_module(grammar._hashed, io, module, lines=[])
+    try_to_save_module(grammar._hashed, io, module, lines=[])
     assert load_module(grammar._hashed, io) == module
 
     os.unlink(_get_hashed_path(grammar._hashed, path))
@@ -144,11 +150,8 @@ def test_cache_last_used_update(diff_cache, use_file_io):
     node_cache_item = next(iter(parser_cache.values()))[p]
     assert now < node_cache_item.last_used < time.time()
 
-@pytest.mark.skipif(
+
-    is_pypy, 
+@skip_pypy
-    reason="pickling in pypy is slow, since we don't pickle,"
-           "we never go into path of auto-collecting garbage"
-)
 def test_inactive_cache(tmpdir, isolated_parso_cache):
     parser_cache.clear()
     test_subjects = "abcdef"
@@ -159,12 +162,12 @@ def test_inactive_cache(tmpdir, isolated_parso_cache):
     paths = os.listdir(raw_cache_path)
     a_while_ago = time.time() - _CACHED_FILE_MAXIMUM_SURVIVAL
     old_paths = set()
-    for path in paths[:len(test_subjects) // 2]: # make certain number of paths old
+    for path in paths[:len(test_subjects) // 2]:  # make certain number of paths old
         os.utime(os.path.join(raw_cache_path, path), (a_while_ago, a_while_ago))
         old_paths.add(path)
     # nothing should be cleared while the lock is on
     assert os.path.exists(_get_cache_clear_lock().path)
-    _remove_cache_and_update_lock() # it shouldn't clear anything
+    _remove_cache_and_update_lock()  # it shouldn't clear anything
     assert len(os.listdir(raw_cache_path)) == len(test_subjects)
     assert old_paths.issubset(os.listdir(raw_cache_path))
 
@@ -172,3 +175,17 @@ def test_inactive_cache(tmpdir, isolated_parso_cache):
     _remove_cache_and_update_lock()
     assert len(os.listdir(raw_cache_path)) == len(test_subjects) // 2
     assert not old_paths.intersection(os.listdir(raw_cache_path))
+
+
+@skip_pypy
+def test_permission_error(monkeypatch):
+    def save(*args, **kwargs):
+        was_called[0] = True  # Python 2... Use nonlocal instead
+        raise PermissionError
+
+    was_called = [False]
+
+    monkeypatch.setattr(cache, '_save_to_file_system', save)
+    with pytest.warns(Warning):
+        parse(path=__file__, cache=True, diff_cache=True)
+    assert was_called[0]