VimPlug/parso
1
0
Fork 0
mirror of https://github.com/davidhalter/parso.git synced 2025-12-06 21:04:29 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Actually start using the stuff we defined in grammar.

Browse Source
This commit is contained in:
Dave Halter
2017-05-25 13:38:49 -04:00
parent 333046c253
commit e7ea01fb35
2 changed files with 12 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
parso/__init__.py
View File

@@ -18,7 +18,7 @@ PythonNode(simple_stmt, [PythonNode(arith_expr, [...]), <Newline: ''>])
"""
from parso.parser import ParserSyntaxError
from parso.grammar import create_grammar, load_grammar
from parso.grammar import Grammar, load_grammar
__version__ = '0.0.3'

25
parso/grammar.py
View File

@@ -4,7 +4,7 @@ import os
from parso._compatibility import FileNotFoundError
from parso.pgen2.pgen import generate_grammar
from parso.utils import splitlines, source_to_unicode
from parso.python.parser import Parser, remove_last_newline
from parso.python.parser import remove_last_newline
from parso.python.diff import DiffParser
from parso.tokenize import generate_tokens
from parso.cache import parser_cache, load_module, save_module
@@ -15,12 +15,16 @@ _loaded_grammars = {}
class Grammar(object):
def __init__(self, bnf_text, tokenizer, parser, diff_parser=None):
self._pgen_grammar = generate_grammar(bnf_text)
"""
:param text: A BNF representation of your grammar.
"""
def __init__(self, text, tokenizer=generate_tokens, parser=BaseParser,
diff_parser=None):
self._pgen_grammar = generate_grammar(text)
self._parser = parser
self._tokenizer = tokenizer
self._diff_parser = diff_parser
self._sha256 = hashlib.sha256(bnf_text.encode("utf-8")).hexdigest()
self._sha256 = hashlib.sha256(text.encode("utf-8")).hexdigest()
def parse(self, code=None, **kwargs):
"""
@@ -106,9 +110,9 @@ class Grammar(object):
tokenize_lines[-1] += '\n'
tokenize_lines.append('')
tokens = generate_tokens(tokenize_lines, use_exact_op_types=True)
tokens = self._tokenizer(tokenize_lines, use_exact_op_types=True)
p = Parser(self._pgen_grammar, error_recovery=error_recovery, start_symbol=start_symbol)
p = self._parser(self._pgen_grammar, error_recovery=error_recovery, start_symbol=start_symbol)
root_node = p.parse(tokens=tokens)
if added_newline:
remove_last_newline(root_node)
@@ -124,13 +128,6 @@ class Grammar(object):
return '<%s:%s>' % (self.__class__.__name__, txt)
def create_grammar(text, tokenizer=generate_tokens, parser=BaseParser):
"""
:param text: A BNF representation of your grammar.
"""
return Grammar(text, tokenizer=tokenizer, parser=parser)
def load_grammar(version=None):
"""
Loads a Python grammar. The default version is always the latest.
@@ -156,7 +153,7 @@ def load_grammar(version=None):
try:
with open(path) as f:
bnf_text = f.read()
grammar = create_grammar(bnf_text, parser=PythonParser)
grammar = Grammar(bnf_text, parser=PythonParser)
return _loaded_grammars.setdefault(path, grammar)
except FileNotFoundError:
# Just load the default if the file does not exist.