Add a more API focused grammar.

This makes it so we don't have to expose all the details of a pgen grammar to the user.

parso/__init__.py

@@ -1,11 +1,57 @@
+import os
+
+from parso._compatibility import FileNotFoundError
 from parso.parser import ParserSyntaxError
-from parso.pgen2.pgen import generate_grammar
 from parso import python
+from parso import grammar
+from parso.tokenize import generate_tokens
+from parso.python.parser import Parser
 
 
 __version__ = '0.0.1'
 
+_loaded_grammars = {}
+
 
 def parse(grammar, code):
     raise NotImplementedError
     Parser(grammar, code)
+
+
+def create_grammar(text, tokenizer=generate_tokens):
+    """
+    :param text: A BNF representation of your grammar.
+    """
+    return grammar.Grammar(text, tokenizer, parser=None)
+
+
+def load_python_grammar(version=None):
+    """
+    Loads a Python grammar. The default version is always the latest.
+
+    If you need support for a specific version, please use e.g.
+    `version='3.3'`.
+    """
+    if version is None:
+        version = '3.6'
+
+    if version in ('3.2', '3.3'):
+        version = '3.4'
+    elif version == '2.6':
+        version = '2.7'
+
+    file = 'python/grammar' + version + '.txt'
+
+    global _loaded_grammars
+    path = os.path.join(os.path.dirname(__file__), file)
+    try:
+        return _loaded_grammars[path]
+    except KeyError:
+        try:
+            with open(path) as f:
+                bnf_text = f.read()
+            grammar = create_grammar(bnf_text)
+            return _loaded_grammars.setdefault(path, grammar)
+        except FileNotFoundError:
+            # Just load the default if the file does not exist.
+            return load_python_grammar()

parso/grammar.py

@@ -0,0 +1,17 @@
+import hashlib
+
+from parso.pgen2.pgen import generate_grammar
+
+
+class Grammar(object):
+    def __init__(self, bnf_text, tokenizer, parser, diff_parser=None):
+        self._pgen_grammar = generate_grammar(bnf_text)
+        self._parser = parser
+        self._tokenizer = tokenizer
+        self._diff_parser = diff_parser
+        self.sha256 = hashlib.sha256(bnf_text.encode("utf-8")).hexdigest()
+
+    def __repr__(self):
+        labels = self._pgen_grammar.symbol2number.values()
+        txt = ' '.join(list(labels)[:3]) + ' ...'
+        return '<%s:%s>' % (self.__class__.__name__, txt)

parso/pgen2/grammar.py

@@ -17,8 +17,6 @@ fallback token code OP, but the parser needs the actual token code.
 """
 
 import pickle
-import hashlib
-
 
 
 class Grammar(object):
@@ -85,7 +83,6 @@ class Grammar(object):
         self.tokens = {}
         self.symbol2label = {}
         self.start = 256
-        self.sha256 = hashlib.sha256(bnf_text.encode("utf-8")).hexdigest()
 
     def dump(self, filename):
         """Dump the grammar tables to a pickle file."""

parso/python/__init__.py

@@ -1,52 +1,13 @@
 """
 Parsers for Python
 """
-import os
-
 from parso.utils import splitlines, source_to_unicode
-from parso._compatibility import FileNotFoundError
-from parso.pgen2.pgen import generate_grammar
 from parso.python.parser import Parser, remove_last_newline
 from parso.python.diff import DiffParser
 from parso.tokenize import generate_tokens
 from parso.cache import parser_cache, load_module, save_module
 
 
-_loaded_grammars = {}
-
-
-def load_grammar(version=None):
-    """
-    Loads a Python grammar. The default version is always the latest.
-
-    If you need support for a specific version, please use e.g.
-    `version='3.3'`.
-    """
-    if version is None:
-        version = '3.6'
-
-    if version in ('3.2', '3.3'):
-        version = '3.4'
-    elif version == '2.6':
-        version = '2.7'
-
-    file = 'grammar' + version + '.txt'
-
-    global _loaded_grammars
-    path = os.path.join(os.path.dirname(__file__), file)
-    try:
-        return _loaded_grammars[path]
-    except KeyError:
-        try:
-            with open(path) as f:
-                bnf_text = f.read()
-            grammar = generate_grammar(bnf_text)
-            return _loaded_grammars.setdefault(path, grammar)
-        except FileNotFoundError:
-            # Just load the default if the file does not exist.
-            return load_grammar()
-
-
 def parse(code=None, **kwargs):
     """
     If you want to parse a Python file you want to start here, most likely.
@@ -78,7 +39,8 @@ def parse(code=None, **kwargs):
             raise TypeError("Please provide either code or a path.")
 
         if grammar is None:
-            grammar = load_grammar()
+            from parso import load_python_grammar
+            grammar = load_python_grammar()
 
         if cache and code is None and path is not None:
             # With the current architecture we cannot load from cache if the
@@ -124,7 +86,7 @@ def parse(code=None, **kwargs):
 
         tokens = generate_tokens(tokenize_lines, use_exact_op_types=True)
 
-        p = Parser(grammar, error_recovery=error_recovery, start_symbol=start_symbol)
+        p = Parser(grammar._pgen_grammar, error_recovery=error_recovery, start_symbol=start_symbol)
         root_node = p.parse(tokens=tokens)
         if added_newline:
             remove_last_newline(root_node)

test/test_cache.py

@@ -8,7 +8,7 @@ import pytest
 
 from parso.cache import _NodeCacheItem, save_module, load_module, \
     _get_hashed_path, parser_cache, _load_from_file_system, _save_to_file_system
-from parso.python import load_grammar
+from parso import load_python_grammar
 from parso import cache
 
 
@@ -37,7 +37,7 @@ def test_modulepickling_change_cache_dir(tmpdir):
     path_1 = 'fake path 1'
     path_2 = 'fake path 2'
 
-    grammar = load_grammar()
+    grammar = load_python_grammar()
     _save_to_file_system(grammar, path_1, item_1, cache_path=dir_1)
     parser_cache.clear()
     cached = load_stored_item(grammar, path_1, item_1, cache_path=dir_1)
@@ -69,7 +69,7 @@ def test_modulepickling_simulate_deleted_cache(tmpdir):
 
     __ https://developer.apple.com/library/content/documentation/FileManagement/Conceptual/FileSystemProgrammingGuide/FileSystemOverview/FileSystemOverview.html
     """
-    grammar = load_grammar()
+    grammar = load_python_grammar()
     module = 'fake parser'
 
     # Create the file

test/test_diff_parser.py

@@ -5,7 +5,7 @@ import pytest
 
 from parso.utils import splitlines
 from parso import cache
-from parso.python import load_grammar
+from parso import load_python_grammar
 from parso.python.diff import DiffParser
 from parso.python import parse
 
@@ -40,7 +40,7 @@ def _assert_valid_graph(node):
 
 
 class Differ(object):
-    grammar = load_grammar()
+    grammar = load_python_grammar()
 
     def initialize(self, code):
         logging.debug('differ: initialize')
@@ -52,7 +52,7 @@ class Differ(object):
     def parse(self, code, copies=0, parsers=0, expect_error_leaves=False):
         logging.debug('differ: parse copies=%s parsers=%s', copies, parsers)
         lines = splitlines(code, keepends=True)
-        diff_parser = DiffParser(self.grammar, self.module)
+        diff_parser = DiffParser(self.grammar._pgen_grammar, self.module)
         new_module = diff_parser.update(self.lines, lines)
         self.lines = lines
         assert code == new_module.get_code()

test/test_parser.py

@@ -5,7 +5,8 @@ from textwrap import dedent
 import pytest
 
 from parso._compatibility import u, py_version
-from parso.python import parse, load_grammar
+from parso.python import parse
+from parso import load_python_grammar
 from parso.python import tree
 from parso.utils import splitlines
 
@@ -112,7 +113,7 @@ def test_param_splitting():
     """
     def check(src, result):
         # Python 2 tuple params should be ignored for now.
-        grammar = load_grammar('%s.%s' % sys.version_info[:2])
+        grammar = load_python_grammar('%s.%s' % sys.version_info[:2])
         m = parse(src, grammar=grammar)
         if py_version >= 30:
             assert not list(m.iter_funcdefs())
@@ -160,10 +161,10 @@ def test_python3_octal():
 def test_load_newer_grammar():
     # This version shouldn't be out for a while, but if we somehow get this it
     # should just take the latest Python grammar.
-    load_grammar('15.8')
+    load_python_grammar('15.8')
     # The same is true for very old grammars (even though this is probably not
     # going to be an issue.
-    load_grammar('1.5')
+    load_python_grammar('1.5')
 
 
 @pytest.mark.parametrize('code', ['foo "', 'foo """\n', 'foo """\nbar'])

test/test_pgen2.py

@@ -9,14 +9,15 @@ test_grammar.py files from both Python 2 and Python 3.
 from textwrap import dedent
 
 from parso._compatibility import py_version
-from parso.python import parse as _parse, load_grammar
+from parso import load_python_grammar
+from parso.python import parse as _parse
 from parso import ParserSyntaxError
 import pytest
 
 
 def parse(code, version='3.4'):
     code = dedent(code) + "\n\n"
-    grammar = load_grammar(version=version)
+    grammar = load_python_grammar(version=version)
     return _parse(code, grammar=grammar, error_recovery=False)