fixed some bugs within fast_parser and added an option 'settings.fast_parser' to turn on fast parsing.

jedi/evaluate.py

@@ -889,10 +889,10 @@ def get_names_for_scope(scope, position=None, star_search=True,
         # Ignore the Flows, because the classes and functions care for that.
         # InstanceElement of Class is ignored, if it is not the start scope.
         if not (scope != non_flow and scope.isinstance(parsing.Class)
-                    or scope.isinstance(parsing.Flow)
+                or scope.isinstance(parsing.Flow)
-                    or scope.isinstance(Instance)
+                or scope.isinstance(Instance) and non_flow.isinstance(Function)
-                        and non_flow.isinstance(Function)
+                or isinstance(scope, parsing.SubModule) and scope.parent
-                    ):
+                ):
             try:
                 if isinstance(scope, Instance):
                     for g in scope.scope_generator():
@@ -1100,6 +1100,9 @@ def get_scopes_for_name(scope, name_str, position=None, search_global=False,
                 if isinstance(p, InstanceElement) \
                             and isinstance(p.var, parsing.Class):
                     p = p.var
+                if isinstance(p, parsing.SubModule) and p.parent is not None:
+                    p = p.parent
+
                 if name_str == name.get_code() and p not in break_scopes:
                     r, no_break_scope = process(name)
                     if is_goto:

jedi/fast_parser.py

@@ -2,6 +2,7 @@ import re
 import operator
 from functools import reduce
 
+import common
 import parsing
 from _compatibility import use_metaclass
 
@@ -10,25 +11,19 @@ parser_cache = {}
 
 class Module(parsing.Simple, parsing.Module):
     def __init__(self, parsers):
-        super(Module, self).__init__((0,0))
+        super(Module, self).__init__((1,0))
         self.parsers = parsers
         self.reset_caches()
 
-        self.subscopes = []
-        self.imports = []
-        self.statements = []
-        self.asserts = []
-
     def reset_caches(self):
         """ This module does a whole lot of caching, because it uses different
         parsers. """
         self.cache = {}
-        self.modules = [p.module for p in self.parsers]
 
     def _get(self, name, operation, *args, **kwargs):
         key = (name, args, frozenset(kwargs.items()))
         if key not in self.cache:
-            objs = (getattr(m, name)(*args, **kwargs) for m in self.modules)
+            objs = (getattr(p.module, name)(*args, **kwargs) for p in self.parsers)
             self.cache[key] = reduce(operation, objs)
         return self.cache[key]
 
@@ -43,13 +38,16 @@ class Module(parsing.Simple, parsing.Module):
                       'imports': operator.add,
                       'statements': operator.add,
                       'imports': operator.add,
-                      'asserts': operator.add
+                      'asserts': operator.add,
+                      'global_vars': operator.add
                      }
         if name in operators:
             return lambda *args, **kwargs: self._get(name, operators[name],
                                                         *args, **kwargs)
         elif name in properties:
             return self._get(name, properties[name])
+        else:
+            raise AttributeError()
 
     def get_statement_for_position(self, pos):
         key = 'get_statement_for_position', pos
@@ -59,19 +57,48 @@ class Module(parsing.Simple, parsing.Module):
                 if s:
                     self.cache[key] = s
                     break
+            else:
+                self.cache[key] = None
+        return self.cache[key]
+
+    @property
+    def used_names(self):
+        key = 'used_names'
+        if key not in self.cache:
+            dct = {}
+            for p in self.parsers:
+                for k, statement_set in p.module.used_names.items():
+                    if k in dct:
+                        dct[k] |= statement_set
+                    else:
+                        dct[k] = set(statement_set)
+
+            self.cache[key] = dct
         return self.cache[key]
 
     @property
     def docstr(self):
-        return self.modules[0].docstr
+        return self.parsers[0].module.docstr
 
     @property
     def name(self):
-        return self.modules[0].name
+        return self.parsers[0].module.name
+
+    @property
+    def path(self):
+        return self.parsers[0].module.path
 
     @property
     def is_builtin(self):
-        return self.modules[0].is_builtin
+        return self.parsers[0].module.is_builtin
+
+    @property
+    def end_pos(self):
+        return self.parsers[-1].module.end_pos
+
+    @end_pos.setter
+    def end_pos(self, value):
+        pass  # just ignore, end_pos is not important
 
     def __repr__(self):
         return "<%s: %s@%s-%s>" % (type(self).__name__, self.name,
@@ -82,7 +109,8 @@ class CachedFastParser(type):
     """ This is a metaclass for caching `FastParser`. """
     def __call__(self, code, module_path=None, user_position=None):
         if module_path is None or module_path not in parser_cache:
-            p = super(CachedFastParser, self).__call__(code, module_path)
+            p = super(CachedFastParser, self).__call__(code, module_path,
+                                                            user_position)
             parser_cache[module_path] = p
         else:
             p = parser_cache[module_path]
@@ -98,16 +126,20 @@ class FastParser(use_metaclass(CachedFastParser)):
 
         self.parsers = []
         self.module = Module(self.parsers)
-        self._parse(code)
-
         self.reset_caches()
 
+        self._parse(code)
+
     @property
     def user_scope(self):
         if self._user_scope is None:
             for p in self.parsers:
                 if p.user_scope:
                     self._user_scope = p.user_scope
+                    break
+
+        if isinstance(self._user_scope, parsing.SubModule):
+            self._user_scope = self.module
         return self._user_scope
 
     @property
@@ -120,19 +152,23 @@ class FastParser(use_metaclass(CachedFastParser)):
 
     def update(self, code, user_position=None):
         self.user_position = user_position
-        self._parse(code)
         self.reset_caches()
+        self.parsers = []  # TODO remove
+        self._parse(code)
 
     def _parse(self, code):
-        parts = re.split(r'\n(?:def|class).*?(?!\n(?:def|class))')
+        parts = re.findall(r'(?:\n(?:def|class)|^).*?(?=\n(?:def|class)|$)',
+                            code, re.DOTALL)
         line_offset = 0
         for p in parts:
             lines = p.count('\n')
             p = parsing.PyFuzzyParser(p, self.module_path, self.user_position,
                                 line_offset=line_offset, stop_on_scope=True)
+            p.module.parent = self.module
             line_offset += lines
             self.parsers.append(p)
 
     def reset_caches(self):
         self._user_scope = None
+        self._user_stmt = None
         self.module.reset_caches()

jedi/modules.py

@@ -10,6 +10,7 @@ import time
 
 import cache
 import parsing
+import fast_parser
 import builtin
 import debug
 import settings
@@ -71,8 +72,12 @@ class ModuleWithCursor(Module):
             # Call the parser already here, because it will be used anyways.
             # Also, the position is here important (which will not be used by
             # default), therefore fill the cache here.
-            self._parser = parsing.PyFuzzyParser(self.source, self.path,
+            if settings.fast_parser:
-                                                                self.position)
+                self._parser = fast_parser.FastParser(self.source, self.path,
+                                                            self.position)
+            else:
+                self._parser = parsing.PyFuzzyParser(self.source, self.path,
+                                                            self.position)
             if self.path is not None:
                 builtin.CachedModule.cache[self.path] = time.time(), \
                                                         self._parser

jedi/parsing.py

@@ -26,7 +26,8 @@ of a statement.
 
 All those classes are being generated by PyFuzzyParser, which takes python text
 as input and ignores just all the non-python stuff. Basically you could feed it
-a perl script, and it should still work (which means throw no error.
+a perl script, and it should still work (which means throw no error).
+TODO remove docstr params from Scope.__init__()
 """
 from _compatibility import (next, literal_eval, StringIO,
                             property, cleandoc, Python3Method)
@@ -252,7 +253,7 @@ class SubModule(Scope, Module):
         self.used_names = {}
         self.temp_used_names = []
         # this may be changed depending on fast_parser
-        self.line_offset = 0
+        self._line_offset = 0
 
     def add_global(self, name):
         """
@@ -1596,7 +1597,7 @@ class PyFuzzyParser(object):
             raise
 
         if self.user_position and (self.start_pos[0] == self.user_position[0]
-                        or self.user_scope is None
+                            or self.user_scope is None
                             and self.start_pos[0] >= self.user_position[0]):
             debug.dbg('user scope found [%s] = %s' % \
                     (self.parserline.replace('\n', ''), repr(self.scope)))

jedi/settings.py

@@ -14,6 +14,12 @@ add_dot_after_module = False
 # Removed it again, because in VIM that is not very practical.
 add_bracket_after_function = False
 
+# ----------------
+# parser
+# ----------------
+
+# Use the fast parser, may cause problems sometimes.
+fast_parser = False
 
 # ----------------
 # dynamic stuff