diff --git a/parso/cache.py b/parso/cache.py index 1b61623..2c6493e 100644 --- a/parso/cache.py +++ b/parso/cache.py @@ -78,21 +78,26 @@ class _NodeCacheItem(object): self.change_time = change_time -def load_module(hashed_grammar, path, cache_path=None): +def load_module(hashed_grammar, file_io, cache_path=None): """ Returns a module or None, if it fails. """ try: - p_time = os.path.getmtime(path) + p_time = file_io.get_last_modified() except FileNotFoundError: return None try: - module_cache_item = parser_cache[hashed_grammar][path] + module_cache_item = parser_cache[hashed_grammar][file_io.path] if p_time <= module_cache_item.change_time: return module_cache_item.node except KeyError: - return _load_from_file_system(hashed_grammar, path, p_time, cache_path=cache_path) + return _load_from_file_system( + hashed_grammar, + file_io.path, + p_time, + cache_path=cache_path + ) def _load_from_file_system(hashed_grammar, path, p_time, cache_path=None): @@ -123,9 +128,10 @@ def _load_from_file_system(hashed_grammar, path, p_time, cache_path=None): return module_cache_item.node -def save_module(hashed_grammar, path, module, lines, pickling=True, cache_path=None): +def save_module(hashed_grammar, file_io, module, lines, pickling=True, cache_path=None): + path = file_io.path try: - p_time = None if path is None else os.path.getmtime(path) + p_time = None if path is None else file_io.get_last_modified() except OSError: p_time = None pickling = False diff --git a/parso/file_io.py b/parso/file_io.py new file mode 100644 index 0000000..dda48bb --- /dev/null +++ b/parso/file_io.py @@ -0,0 +1,26 @@ +import os + + +class FileIO: + def __init__(self, path): + self.path = path + + def read(self): + with open(self.path) as f: + return f.read() + + def get_last_modified(self): + """ + Returns float - timestamp + Might raise FileNotFoundError + """ + return os.path.getmtime(self.path) + + +class KnownContentFileIO(FileIO): + def __init__(self, path, content): + super(KnownContentFileIO, self).__init__(path) + self._content = content + + def read(self): + return self._content diff --git a/parso/grammar.py b/parso/grammar.py index caf5001..938439d 100644 --- a/parso/grammar.py +++ b/parso/grammar.py @@ -12,6 +12,7 @@ from parso.parser import BaseParser from parso.python.parser import Parser as PythonParser from parso.python.errors import ErrorFinderConfig from parso.python import pep8 +from parso.file_io import FileIO, KnownContentFileIO _loaded_grammars = {} @@ -77,14 +78,14 @@ class Grammar(object): def _parse(self, code=None, error_recovery=True, path=None, start_symbol=None, cache=False, diff_cache=False, - cache_path=None, start_pos=(1, 0)): + cache_path=None, file_io=None, start_pos=(1, 0)): """ Wanted python3.5 * operator and keyword only arguments. Therefore just wrap it all. start_pos here is just a parameter internally used. Might be public sometime in the future. """ - if code is None and path is None: + if code is None and path is None and file_io is None: raise TypeError("Please provide either code or a path.") if start_symbol is None: @@ -93,15 +94,18 @@ class Grammar(object): if error_recovery and start_symbol != 'file_input': raise NotImplementedError("This is currently not implemented.") - if cache and path is not None: - module_node = load_module(self._hashed, path, cache_path=cache_path) + if file_io is None: + if code is None: + file_io = FileIO(path) + else: + file_io = KnownContentFileIO(path, code) + + if cache and file_io.path is not None: + module_node = load_module(self._hashed, file_io, cache_path=cache_path) if module_node is not None: return module_node - if code is None: - with open(path, 'rb') as f: - code = f.read() - + code = file_io.read() code = python_bytes_to_unicode(code) lines = split_lines(code, keepends=True) @@ -110,7 +114,7 @@ class Grammar(object): raise TypeError("You have to define a diff parser to be able " "to use this option.") try: - module_cache_item = parser_cache[self._hashed][path] + module_cache_item = parser_cache[self._hashed][file_io.path] except KeyError: pass else: @@ -125,7 +129,7 @@ class Grammar(object): old_lines=old_lines, new_lines=lines ) - save_module(self._hashed, path, new_node, lines, + save_module(self._hashed, file_io, new_node, lines, # Never pickle in pypy, it's slow as hell. pickling=cache and not is_pypy, cache_path=cache_path) @@ -141,7 +145,7 @@ class Grammar(object): root_node = p.parse(tokens=tokens) if cache or diff_cache: - save_module(self._hashed, path, root_node, lines, + save_module(self._hashed, file_io, root_node, lines, # Never pickle in pypy, it's slow as hell. pickling=cache and not is_pypy, cache_path=cache_path) diff --git a/test/test_cache.py b/test/test_cache.py index a7cd70c..7fef203 100644 --- a/test/test_cache.py +++ b/test/test_cache.py @@ -10,6 +10,7 @@ from parso.cache import _NodeCacheItem, save_module, load_module, \ _get_hashed_path, parser_cache, _load_from_file_system, _save_to_file_system from parso import load_grammar from parso import cache +from parso import file_io @pytest.fixture() @@ -76,12 +77,13 @@ def test_modulepickling_simulate_deleted_cache(tmpdir): path = tmpdir.dirname + '/some_path' with open(path, 'w'): pass + io = file_io.FileIO(path) - save_module(grammar._hashed, path, module, []) - assert load_module(grammar._hashed, path) == module + save_module(grammar._hashed, io, module, lines=[]) + assert load_module(grammar._hashed, io) == module unlink(_get_hashed_path(grammar._hashed, path)) parser_cache.clear() - cached2 = load_module(grammar._hashed, path) + cached2 = load_module(grammar._hashed, io) assert cached2 is None