Added a grammar param to the parser.

jedi/parser/__init__.py

@@ -15,16 +15,14 @@ within the statement. This lowers memory usage and cpu time and reduces the
 complexity of the ``Parser`` (there's another parser sitting inside
 ``Statement``, which produces ``Array`` and ``Call``).
 """
-import keyword
 import logging
+import os
 
-from jedi._compatibility import next, unicode
-from jedi import debug
+from jedi._compatibility import next
 from jedi import common
 from jedi.parser import tree as pt
 from jedi.parser import tokenize
 from jedi.parser import pytree
-from jedi.parser.pgen2 import Driver
 from jedi.parser import pgen2
 
 OPERATOR_KEYWORDS = 'and', 'for', 'if', 'else', 'in', 'is', 'lambda', 'not', 'or'
@@ -33,20 +31,33 @@ STATEMENT_KEYWORDS = 'assert', 'del', 'global', 'nonlocal', 'raise', \
     'return', 'yield', 'pass', 'continue', 'break'
 
 
+_loaded_grammars = {}
+
+
+def load_grammar(file):
+    global _loaded_grammars
+    path = os.path.join(os.path.dirname(__file__), file) + '.txt'
+    try:
+        return _loaded_grammars[path]
+    except KeyError:
+        return _loaded_grammars.setdefault(path, pgen2.load_grammar(path))
+
+
 class Parser(object):
     """
     This class is used to parse a Python file, it then divides them into a
     class structure of different scopes.
 
-    :param source: The codebase for the parser.
-    :type source: str
+    :param grammar: The grammar object of pgen2. Loaded by load_grammar.
+    :param source: The codebase for the parser. Must be unicode.
     :param module_path: The path of the module in the file system, may be None.
     :type module_path: str
-    :param no_docstr: If True, a string at the beginning is not a docstr.
     :param top_module: Use this module as a parent instead of `self.module`.
     """
-    def __init__(self, source, module_path=None, no_docstr=False,
-                 tokenizer=None, top_module=None):
+    def __init__(self, grammar, source, module_path=None, tokenizer=None):
+        """
+        This is the way I imagine a parser describing the init function
+        """
 
         if not source.endswith('\n'):
             source += '\n'
@@ -90,8 +101,8 @@ class Parser(object):
         self.used_names = {}
         self.scope_names_stack = [{}]
         logger = logging.getLogger("Jedi-Parser")
-        d = Driver(pytree.python_grammar, self.convert_node, self.convert_leaf,
-                   self.error_recovery, logger=logger)
+        d = pgen2.Driver(grammar, self.convert_node,
+                         self.convert_leaf, self.error_recovery, logger=logger)
         self.module = d.parse_string(source).get_parent_until()
 
         self.module.used_names = self.used_names

jedi/parser/fast.py

@@ -56,13 +56,13 @@ class Module(pr.Module, pr.Simple):
 
 class CachedFastParser(type):
     """ This is a metaclass for caching `FastParser`. """
-    def __call__(self, source, module_path=None):
+    def __call__(self, grammar, source, module_path=None):
         if not settings.fast_parser:
-            return Parser(source, module_path)
+            return Parser(grammar, source, module_path)
 
         pi = cache.parser_cache.get(module_path, None)
         if pi is None or isinstance(pi.parser, Parser):
-            p = super(CachedFastParser, self).__call__(source, module_path)
+            p = super(CachedFastParser, self).__call__(grammar, source, module_path)
         else:
             p = pi.parser  # pi is a `cache.ParserCacheItem`
             p.update(source)
@@ -186,8 +186,9 @@ class FastParser(use_metaclass(CachedFastParser)):
 
     _keyword_re = re.compile('^[ \t]*(def|class|@|%s)' % '|'.join(tokenize.FLOWS))
 
-    def __init__(self, code, module_path=None):
+    def __init__(self, grammar, code, module_path=None):
         # set values like `pr.Module`.
+        self._grammar = grammar
         self.module_path = module_path
 
         self.current_node = None

jedi/parser/pytree.py

@@ -18,6 +18,7 @@ from . import pgen2
 
 # The grammar file
 _GRAMMAR_FILE = os.path.join(os.path.dirname(__file__), "grammar3.4.txt")
+python_grammar = pgen2.load_grammar(_GRAMMAR_FILE)
 
 
 class Symbols(object):

jedi/parser/user_context.py

@@ -202,7 +202,8 @@ class UserContext(object):
 
 
 class UserContextParser(object):
-    def __init__(self, source, path, position, user_context):
+    def __init__(self, grammar, source, path, position, user_context):
+        self._grammar = grammar
         self._source = source
         self._path = path and os.path.abspath(path)
         self._position = position
@@ -211,7 +212,7 @@ class UserContextParser(object):
     @cache.underscore_memoization
     def _parser(self):
         cache.invalidate_star_import_cache(self._path)
-        parser = FastParser(self._source, self._path)
+        parser = FastParser(self._grammar, self._source, self._path)
         # Don't pickle that module, because the main module is changing quickly
         cache.save_parser(self._path, None, parser, pickling=False)
         return parser