Improved grammar loading API.

jedi/parser/pgen2/pgen.py

@@ -12,11 +12,8 @@ from jedi.parser import tokenize
 
 
 class ParserGenerator(object):
-    def __init__(self, filename):
+    def __init__(self, bnf_text):
-        with open(filename) as f:
+        self.generator = tokenize.source_tokens(bnf_text)
-            code = f.read()
-        self.filename = filename
-        self.generator = tokenize.source_tokens(code)
         self.gettoken()  # Initialize lookahead
         self.dfas, self.startsymbol = self.parse()
         self.first = {}  # map from symbol name to set of tokens
@@ -384,6 +381,6 @@ class DFAState(object):
     __hash__ = None  # For Py3 compatibility.
 
 
-def generate_grammar(filename="Grammar.txt"):
+def generate_grammar(bnf_text):
-    p = ParserGenerator(filename)
+    p = ParserGenerator(bnf_text)
     return p.make_grammar()

jedi/parser/python/__init__.py

@@ -9,9 +9,16 @@ from jedi.parser.tokenize import source_tokens
 _loaded_grammars = {}
 
 
-def load_grammar(version='3.6'):
+def load_grammar(version=None):
-    # For now we only support two different Python syntax versions: The latest
+    """
-    # Python 3 and Python 2. This may change.
+    Loads a Python grammar. The default version is always the latest.
+
+    If you need support for a specific version, please use e.g.
+    `version='3.3'`.
+    """
+    if version is None:
+        version = '3.6'
+
     if version in ('3.2', '3.3'):
         version = '3.4'
     elif version == '2.6':
@@ -25,7 +32,10 @@ def load_grammar(version='3.6'):
         return _loaded_grammars[path]
     except KeyError:
         try:
-            return _loaded_grammars.setdefault(path, generate_grammar(path))
+            with open(path) as f:
+                bnf_text = f.read()
+            grammar = generate_grammar(bnf_text)
+            return _loaded_grammars.setdefault(path, grammar)
         except FileNotFoundError:
             # Just load the default if the file does not exist.
             return load_grammar()
@@ -40,4 +50,4 @@ def parse(source, grammar=None, error_recovery=False):
         parser = ParserWithRecovery
     else:
         parser = Parser
-    parser(grammar, tokens)
+    return parser(grammar, tokens).get_root_node()