From b8b4a02398bb160c3f7c955f5db245efbfe990ba Mon Sep 17 00:00:00 2001 From: David Halter Date: Fri, 14 Dec 2012 18:16:54 +0100 Subject: [PATCH] fixed some bugs within fast_parser and added an option 'settings.fast_parser' to turn on fast parsing. --- jedi/evaluate.py | 11 ++++--- jedi/fast_parser.py | 70 ++++++++++++++++++++++++++++++++++----------- jedi/modules.py | 9 ++++-- jedi/parsing.py | 7 +++-- jedi/settings.py | 6 ++++ 5 files changed, 77 insertions(+), 26 deletions(-) diff --git a/jedi/evaluate.py b/jedi/evaluate.py index f0474552..40ec4a30 100644 --- a/jedi/evaluate.py +++ b/jedi/evaluate.py @@ -889,10 +889,10 @@ def get_names_for_scope(scope, position=None, star_search=True, # Ignore the Flows, because the classes and functions care for that. # InstanceElement of Class is ignored, if it is not the start scope. if not (scope != non_flow and scope.isinstance(parsing.Class) - or scope.isinstance(parsing.Flow) - or scope.isinstance(Instance) - and non_flow.isinstance(Function) - ): + or scope.isinstance(parsing.Flow) + or scope.isinstance(Instance) and non_flow.isinstance(Function) + or isinstance(scope, parsing.SubModule) and scope.parent + ): try: if isinstance(scope, Instance): for g in scope.scope_generator(): @@ -1100,6 +1100,9 @@ def get_scopes_for_name(scope, name_str, position=None, search_global=False, if isinstance(p, InstanceElement) \ and isinstance(p.var, parsing.Class): p = p.var + if isinstance(p, parsing.SubModule) and p.parent is not None: + p = p.parent + if name_str == name.get_code() and p not in break_scopes: r, no_break_scope = process(name) if is_goto: diff --git a/jedi/fast_parser.py b/jedi/fast_parser.py index f4f09431..741325c3 100644 --- a/jedi/fast_parser.py +++ b/jedi/fast_parser.py @@ -2,6 +2,7 @@ import re import operator from functools import reduce +import common import parsing from _compatibility import use_metaclass @@ -10,25 +11,19 @@ parser_cache = {} class Module(parsing.Simple, parsing.Module): def __init__(self, parsers): - super(Module, self).__init__((0,0)) + super(Module, self).__init__((1,0)) self.parsers = parsers self.reset_caches() - self.subscopes = [] - self.imports = [] - self.statements = [] - self.asserts = [] - def reset_caches(self): """ This module does a whole lot of caching, because it uses different parsers. """ self.cache = {} - self.modules = [p.module for p in self.parsers] def _get(self, name, operation, *args, **kwargs): key = (name, args, frozenset(kwargs.items())) if key not in self.cache: - objs = (getattr(m, name)(*args, **kwargs) for m in self.modules) + objs = (getattr(p.module, name)(*args, **kwargs) for p in self.parsers) self.cache[key] = reduce(operation, objs) return self.cache[key] @@ -43,13 +38,16 @@ class Module(parsing.Simple, parsing.Module): 'imports': operator.add, 'statements': operator.add, 'imports': operator.add, - 'asserts': operator.add + 'asserts': operator.add, + 'global_vars': operator.add } if name in operators: return lambda *args, **kwargs: self._get(name, operators[name], *args, **kwargs) elif name in properties: return self._get(name, properties[name]) + else: + raise AttributeError() def get_statement_for_position(self, pos): key = 'get_statement_for_position', pos @@ -59,19 +57,48 @@ class Module(parsing.Simple, parsing.Module): if s: self.cache[key] = s break + else: + self.cache[key] = None + return self.cache[key] + + @property + def used_names(self): + key = 'used_names' + if key not in self.cache: + dct = {} + for p in self.parsers: + for k, statement_set in p.module.used_names.items(): + if k in dct: + dct[k] |= statement_set + else: + dct[k] = set(statement_set) + + self.cache[key] = dct return self.cache[key] @property def docstr(self): - return self.modules[0].docstr + return self.parsers[0].module.docstr @property def name(self): - return self.modules[0].name + return self.parsers[0].module.name + + @property + def path(self): + return self.parsers[0].module.path @property def is_builtin(self): - return self.modules[0].is_builtin + return self.parsers[0].module.is_builtin + + @property + def end_pos(self): + return self.parsers[-1].module.end_pos + + @end_pos.setter + def end_pos(self, value): + pass # just ignore, end_pos is not important def __repr__(self): return "<%s: %s@%s-%s>" % (type(self).__name__, self.name, @@ -82,7 +109,8 @@ class CachedFastParser(type): """ This is a metaclass for caching `FastParser`. """ def __call__(self, code, module_path=None, user_position=None): if module_path is None or module_path not in parser_cache: - p = super(CachedFastParser, self).__call__(code, module_path) + p = super(CachedFastParser, self).__call__(code, module_path, + user_position) parser_cache[module_path] = p else: p = parser_cache[module_path] @@ -98,16 +126,20 @@ class FastParser(use_metaclass(CachedFastParser)): self.parsers = [] self.module = Module(self.parsers) - self._parse(code) - self.reset_caches() + self._parse(code) + @property def user_scope(self): if self._user_scope is None: for p in self.parsers: if p.user_scope: self._user_scope = p.user_scope + break + + if isinstance(self._user_scope, parsing.SubModule): + self._user_scope = self.module return self._user_scope @property @@ -120,19 +152,23 @@ class FastParser(use_metaclass(CachedFastParser)): def update(self, code, user_position=None): self.user_position = user_position - self._parse(code) self.reset_caches() + self.parsers = [] # TODO remove + self._parse(code) def _parse(self, code): - parts = re.split(r'\n(?:def|class).*?(?!\n(?:def|class))') + parts = re.findall(r'(?:\n(?:def|class)|^).*?(?=\n(?:def|class)|$)', + code, re.DOTALL) line_offset = 0 for p in parts: lines = p.count('\n') p = parsing.PyFuzzyParser(p, self.module_path, self.user_position, line_offset=line_offset, stop_on_scope=True) + p.module.parent = self.module line_offset += lines self.parsers.append(p) def reset_caches(self): self._user_scope = None + self._user_stmt = None self.module.reset_caches() diff --git a/jedi/modules.py b/jedi/modules.py index af0aa7b8..5eb2aa75 100644 --- a/jedi/modules.py +++ b/jedi/modules.py @@ -10,6 +10,7 @@ import time import cache import parsing +import fast_parser import builtin import debug import settings @@ -71,8 +72,12 @@ class ModuleWithCursor(Module): # Call the parser already here, because it will be used anyways. # Also, the position is here important (which will not be used by # default), therefore fill the cache here. - self._parser = parsing.PyFuzzyParser(self.source, self.path, - self.position) + if settings.fast_parser: + self._parser = fast_parser.FastParser(self.source, self.path, + self.position) + else: + self._parser = parsing.PyFuzzyParser(self.source, self.path, + self.position) if self.path is not None: builtin.CachedModule.cache[self.path] = time.time(), \ self._parser diff --git a/jedi/parsing.py b/jedi/parsing.py index dd576b45..489ffd0b 100644 --- a/jedi/parsing.py +++ b/jedi/parsing.py @@ -26,7 +26,8 @@ of a statement. All those classes are being generated by PyFuzzyParser, which takes python text as input and ignores just all the non-python stuff. Basically you could feed it -a perl script, and it should still work (which means throw no error. +a perl script, and it should still work (which means throw no error). +TODO remove docstr params from Scope.__init__() """ from _compatibility import (next, literal_eval, StringIO, property, cleandoc, Python3Method) @@ -252,7 +253,7 @@ class SubModule(Scope, Module): self.used_names = {} self.temp_used_names = [] # this may be changed depending on fast_parser - self.line_offset = 0 + self._line_offset = 0 def add_global(self, name): """ @@ -1596,7 +1597,7 @@ class PyFuzzyParser(object): raise if self.user_position and (self.start_pos[0] == self.user_position[0] - or self.user_scope is None + or self.user_scope is None and self.start_pos[0] >= self.user_position[0]): debug.dbg('user scope found [%s] = %s' % \ (self.parserline.replace('\n', ''), repr(self.scope))) diff --git a/jedi/settings.py b/jedi/settings.py index 23f366dc..005c3bca 100644 --- a/jedi/settings.py +++ b/jedi/settings.py @@ -14,6 +14,12 @@ add_dot_after_module = False # Removed it again, because in VIM that is not very practical. add_bracket_after_function = False +# ---------------- +# parser +# ---------------- + +# Use the fast parser, may cause problems sometimes. +fast_parser = False # ---------------- # dynamic stuff