From e7ea01fb35a714b1d9adf54fedf920c0a1b9a454 Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Thu, 25 May 2017 13:38:49 -0400 Subject: [PATCH] Actually start using the stuff we defined in grammar. --- parso/__init__.py | 2 +- parso/grammar.py | 25 +++++++++++-------------- 2 files changed, 12 insertions(+), 15 deletions(-) diff --git a/parso/__init__.py b/parso/__init__.py index f5fe4d7..b1bf888 100644 --- a/parso/__init__.py +++ b/parso/__init__.py @@ -18,7 +18,7 @@ PythonNode(simple_stmt, [PythonNode(arith_expr, [...]), ]) """ from parso.parser import ParserSyntaxError -from parso.grammar import create_grammar, load_grammar +from parso.grammar import Grammar, load_grammar __version__ = '0.0.3' diff --git a/parso/grammar.py b/parso/grammar.py index 6c15755..25a6c3f 100644 --- a/parso/grammar.py +++ b/parso/grammar.py @@ -4,7 +4,7 @@ import os from parso._compatibility import FileNotFoundError from parso.pgen2.pgen import generate_grammar from parso.utils import splitlines, source_to_unicode -from parso.python.parser import Parser, remove_last_newline +from parso.python.parser import remove_last_newline from parso.python.diff import DiffParser from parso.tokenize import generate_tokens from parso.cache import parser_cache, load_module, save_module @@ -15,12 +15,16 @@ _loaded_grammars = {} class Grammar(object): - def __init__(self, bnf_text, tokenizer, parser, diff_parser=None): - self._pgen_grammar = generate_grammar(bnf_text) + """ + :param text: A BNF representation of your grammar. + """ + def __init__(self, text, tokenizer=generate_tokens, parser=BaseParser, + diff_parser=None): + self._pgen_grammar = generate_grammar(text) self._parser = parser self._tokenizer = tokenizer self._diff_parser = diff_parser - self._sha256 = hashlib.sha256(bnf_text.encode("utf-8")).hexdigest() + self._sha256 = hashlib.sha256(text.encode("utf-8")).hexdigest() def parse(self, code=None, **kwargs): """ @@ -106,9 +110,9 @@ class Grammar(object): tokenize_lines[-1] += '\n' tokenize_lines.append('') - tokens = generate_tokens(tokenize_lines, use_exact_op_types=True) + tokens = self._tokenizer(tokenize_lines, use_exact_op_types=True) - p = Parser(self._pgen_grammar, error_recovery=error_recovery, start_symbol=start_symbol) + p = self._parser(self._pgen_grammar, error_recovery=error_recovery, start_symbol=start_symbol) root_node = p.parse(tokens=tokens) if added_newline: remove_last_newline(root_node) @@ -124,13 +128,6 @@ class Grammar(object): return '<%s:%s>' % (self.__class__.__name__, txt) -def create_grammar(text, tokenizer=generate_tokens, parser=BaseParser): - """ - :param text: A BNF representation of your grammar. - """ - return Grammar(text, tokenizer=tokenizer, parser=parser) - - def load_grammar(version=None): """ Loads a Python grammar. The default version is always the latest. @@ -156,7 +153,7 @@ def load_grammar(version=None): try: with open(path) as f: bnf_text = f.read() - grammar = create_grammar(bnf_text, parser=PythonParser) + grammar = Grammar(bnf_text, parser=PythonParser) return _loaded_grammars.setdefault(path, grammar) except FileNotFoundError: # Just load the default if the file does not exist.