diff --git a/jedi/evaluate/sys_path.py b/jedi/evaluate/sys_path.py index 66c9db59..c3ff7aab 100644 --- a/jedi/evaluate/sys_path.py +++ b/jedi/evaluate/sys_path.py @@ -5,12 +5,11 @@ from jedi.evaluate.site import addsitedir from jedi._compatibility import exec_function, unicode from jedi.parser.python import tree -from jedi.parser import ParserWithRecovery +from jedi.parser.python import parse from jedi.evaluate.cache import memoize_default from jedi import debug from jedi import common from jedi.evaluate.compiled import CompiledObject -from jedi.parser.utils import load_parser, save_parser def get_venv_path(venv): @@ -211,22 +210,15 @@ def sys_path_with_modifications(evaluator, module_context): return list(result) + list(buildout_script_paths) -def _get_paths_from_buildout_script(evaluator, buildout_script): - def load(buildout_script): - try: - with open(buildout_script, 'rb') as f: - source = common.source_to_unicode(f.read()) - except IOError: - debug.dbg('Error trying to read buildout_script: %s', buildout_script) - return - - p = ParserWithRecovery(evaluator.grammar, source, buildout_script) - save_parser(evaluator.grammar, buildout_script, p) - return p.get_root_node() - - cached = load_parser(evaluator.grammar, buildout_script) - module_node = cached and cached.module or load(buildout_script) - if module_node is None: +def _get_paths_from_buildout_script(evaluator, buildout_script_path): + try: + module_node = parse( + path=buildout_script_path, + grammar=evaluator.grammar, + cache=True + ) + except IOError: + debug.warning('Error trying to read buildout_script: %s', buildout_script_path) return from jedi.evaluate.representation import ModuleContext diff --git a/jedi/parser/python/__init__.py b/jedi/parser/python/__init__.py index 9dfa7bde..dfd8e734 100644 --- a/jedi/parser/python/__init__.py +++ b/jedi/parser/python/__init__.py @@ -8,6 +8,7 @@ from jedi.parser.pgen2.pgen import generate_grammar from jedi.parser.python.parser import Parser, ParserWithRecovery, \ _remove_last_newline from jedi.parser.tokenize import source_tokens +from jedi.parser import utils _loaded_grammars = {} @@ -46,7 +47,7 @@ def load_grammar(version=None): def parse(code=None, path=None, grammar=None, error_recovery=True, - start_symbol='file_input', cache=False): + start_symbol='file_input', cache=False, diff_cache=False): """ If you want to parse a Python file you want to start here, most likely. @@ -64,20 +65,30 @@ def parse(code=None, path=None, grammar=None, error_recovery=True, :return: A syntax tree node. Typically the module. """ - if start_symbol != 'file_input' and error_recovery: - raise Exception( - 'The start_symbol is only allowed when error recovery is disabled.') + if code is None and path is None: + raise TypeError("Please provide either code or a path.") + + if grammar is None: + grammar = load_grammar() + + if path is not None: + path = os.path.expanduser(path) + + use_cache = cache and path is not None and not code + if use_cache: + # In this case we do actual caching. We just try to load it. + p = utils.load_parser(grammar, path) + if p is not None: + return p.get_root_node() + + if code is None: + with open(path) as f: + code = f.read() added_newline = not code.endswith('\n') if added_newline: code += '\n' - if grammar is None: - grammar = load_grammar() - - if cache and path and not code: - # In this case we do actual caching - path = os.path.expanduser(path) tokens = source_tokens(code, use_exact_op_types=True) kwargs = {} if error_recovery: @@ -90,4 +101,6 @@ def parse(code=None, path=None, grammar=None, error_recovery=True, if added_newline: _remove_last_newline(module) + if use_cache: + utils.save_parser(grammar, path, parser) return module diff --git a/jedi/parser/utils.py b/jedi/parser/utils.py index f74e3e63..2bdb767a 100644 --- a/jedi/parser/utils.py +++ b/jedi/parser/utils.py @@ -119,7 +119,6 @@ class ParserPickling(object): greater than the original pickling time. In which case the pickled parser is not up to date. """ - path = self._clean_path(path) try: pickle_changed_time = self._index[path] except KeyError: @@ -144,7 +143,6 @@ class ParserPickling(object): return parser_cache_item.parser def save_parser(self, grammar, path, parser_cache_item): - path = self._clean_path(path) self.__index = None try: files = self._index @@ -191,9 +189,6 @@ class ParserPickling(object): shutil.rmtree(self._cache_directory()) self.__index = {} - def _clean_path(self, path): - return os.path.expanduser(path) - def _get_hashed_path(self, grammar, path): file_hash = hashlib.sha256(path.encode("utf-8")).hexdigest() return self._get_path('%s-%s.pkl' % (grammar.sha256, file_hash))