From 90edb2d0cfa649fcffcf6cebfdee5b00d54d9e9f Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Sun, 6 Aug 2017 15:32:50 +0200 Subject: [PATCH] Move the iter_errors and normalizer methods to grammar.py. --- conftest.py | 5 ++-- parso/grammar.py | 42 ++++++++++++++++++++++++++-- parso/python/normalizer.py | 2 +- parso/python/tree.py | 7 ----- parso/tree.py | 23 --------------- test/test_file_python_errors.py | 2 +- test/test_normalizer_issues_files.py | 2 +- test/test_pep8.py | 2 +- test/test_python_errors.py | 2 +- 9 files changed, 47 insertions(+), 40 deletions(-) diff --git a/conftest.py b/conftest.py index 2a2504e..45c6ef8 100644 --- a/conftest.py +++ b/conftest.py @@ -126,7 +126,7 @@ class Checker(): print(module.children) def get_error(self, code): - errors = list(self.grammar.parse(code)._iter_errors(self.grammar)) + errors = list(self.grammar.iter_errors(self.grammar.parse(code))) assert bool(errors) != self._is_passing if errors: return errors[0] @@ -139,7 +139,8 @@ class Checker(): def assert_no_error_in_passing(self, code): if self._is_passing: - assert not list(self.grammar.parse(code)._iter_errors(self.grammar)) + module = self.grammar.parse(code) + assert not list(self.grammar.iter_errors(module)) @pytest.fixture diff --git a/parso/grammar.py b/parso/grammar.py index f2aed64..f1e4958 100644 --- a/parso/grammar.py +++ b/parso/grammar.py @@ -9,6 +9,8 @@ from parso.python.tokenize import tokenize_lines, tokenize from parso.cache import parser_cache, load_module, save_module from parso.parser import BaseParser from parso.python.parser import Parser as PythonParser +from parso.python.normalizer import ErrorFinderConfig +from parso.python import pep8 _loaded_grammars = {} @@ -19,6 +21,9 @@ class Grammar(object): :param text: A BNF representation of your grammar. """ + _error_finder_config = None + _default_normalizer_config = pep8.PEP8NormalizerConfig() + def __init__(self, text, tokenizer, parser=BaseParser, diff_parser=None): self._pgen_grammar = generate_grammar(text) self._parser = parser @@ -120,6 +125,34 @@ class Grammar(object): cache_path=cache_path) return root_node + def iter_errors(self, node): + if self._error_normalizer_config is None: + raise ValueError("No error normalizer specified for this grammar.") + + return self._get_normalizer_issues(node, self._error_normalizer_config) + + def _get_normalizer(self, normalizer_config): + if normalizer_config is None: + normalizer_config = self._default_normalizer_config + if normalizer_config is None: + raise ValueError("You need to specify a normalizer, because " + "there's no default normalizer for this tree.") + return normalizer_config.create_normalizer(self) + + def _normalize(self, node, normalizer_config=None): + """ + TODO this is not public, yet. + The returned code will be normalized, e.g. PEP8 for Python. + """ + normalizer = self._get_normalizer(normalizer_config) + return normalizer.walk(node) + + def _get_normalizer_issues(self, node, normalizer_config=None): + normalizer = self._get_normalizer(normalizer_config) + normalizer.walk(node) + return normalizer.issues + + def __repr__(self): labels = self._pgen_grammar.symbol2number.values() txt = ' '.join(list(labels)[:3]) + ' ...' @@ -127,6 +160,9 @@ class Grammar(object): class PythonGrammar(Grammar): + _error_normalizer_config = ErrorFinderConfig() + _parser_cls = PythonParser + def __init__(self, version_info, bnf_text): super(PythonGrammar, self).__init__( bnf_text, @@ -134,14 +170,14 @@ class PythonGrammar(Grammar): parser=PythonParser, diff_parser=DiffParser ) - self._version_int = version_info + self.version_info = version_info def _tokenize_lines(self, lines): - return tokenize_lines(lines, self._version_int) + return tokenize_lines(lines, self.version_info) def _tokenize(self, code): # Used by Jedi. - return tokenize(code, self._version_int) + return tokenize(code, self.version_info) def load_grammar(**kwargs): diff --git a/parso/python/normalizer.py b/parso/python/normalizer.py index 577ed45..1fa08ba 100644 --- a/parso/python/normalizer.py +++ b/parso/python/normalizer.py @@ -257,7 +257,7 @@ class ErrorFinder(Normalizer): def __init__(self, *args, **kwargs): super(ErrorFinder, self).__init__(*args, **kwargs) self._error_dict = {} - self._version = self._grammar._version_int + self._version = self._grammar.version_info def initialize(self, node): from parso.python.tree import search_ancestor diff --git a/parso/python/tree.py b/parso/python/tree.py index c93af25..58b8a46 100644 --- a/parso/python/tree.py +++ b/parso/python/tree.py @@ -30,8 +30,6 @@ import re from parso._compatibility import utf8_repr, unicode from parso.tree import Node, BaseNode, Leaf, ErrorNode, ErrorLeaf, \ search_ancestor -from parso.python import normalizer -from parso.python import pep8 from parso.python.prefix import split_prefix @@ -68,7 +66,6 @@ class PythonMixin(object): Some Python specific utitilies. """ __slots__ = () - default_normalizer_config = pep8.PEP8NormalizerConfig() def get_definition(self): if self.type in ('newline', 'endmarker'): @@ -99,10 +96,6 @@ class PythonMixin(object): return result return None - def _iter_errors(self, grammar): - config = normalizer.ErrorFinderConfig() - return self._get_normalizer_issues(grammar, config) - class PythonLeaf(PythonMixin, Leaf): __slots__ = () diff --git a/parso/tree.py b/parso/tree.py index a47b02f..2a84784 100644 --- a/parso/tree.py +++ b/parso/tree.py @@ -22,7 +22,6 @@ class NodeOrLeaf(object): The base class for nodes and leaves. """ __slots__ = () - default_normalizer_config = None def get_root_node(self): """ @@ -154,28 +153,6 @@ class NodeOrLeaf(object): e.g. a statement. """ - def _get_normalizer(self, normalizer_config, grammar): - if normalizer_config is None: - normalizer_config = self.default_normalizer_config - if normalizer_config is None: - raise ValueError("You need to specify a normalizer, because " - "there's no default normalizer for this tree.") - return normalizer_config.create_normalizer(grammar) - - def _normalize(self, grammar, normalizer_config=None): - """ - TODO this is not public, yet. - The returned code will be normalized, e.g. PEP8 for Python. - """ - normalizer = self._get_normalizer(normalizer_config, grammar) - return normalizer.walk(self) - - def _get_normalizer_issues(self, grammar, normalizer_config=None): - normalizer = self._get_normalizer(normalizer_config, grammar) - normalizer.walk(self) - return normalizer.issues - - class Leaf(NodeOrLeaf): __slots__ = ('value', 'parent', 'line', 'indent', 'prefix') diff --git a/test/test_file_python_errors.py b/test/test_file_python_errors.py index 7d620e1..7083dfe 100644 --- a/test/test_file_python_errors.py +++ b/test/test_file_python_errors.py @@ -19,5 +19,5 @@ def test_on_itself(each_version): path = os.path.dirname(os.path.dirname(__file__)) + '/parso' for file in get_python_files(path): tree = grammar.parse(path=file) - errors = list(tree._iter_errors(grammar)) + errors = list(grammar.iter_errors(tree)) assert not errors diff --git a/test/test_normalizer_issues_files.py b/test/test_normalizer_issues_files.py index c6e7955..eb65bd3 100644 --- a/test/test_normalizer_issues_files.py +++ b/test/test_normalizer_issues_files.py @@ -60,7 +60,7 @@ def test_normalizer_issue(normalizer_issue_case): grammar = parso.load_grammar(version=normalizer_issue_case.python_version) module = grammar.parse(code) - issues = module._get_normalizer_issues(grammar) + issues = grammar._get_normalizer_issues(module) actual = sort(issues) diff = '\n'.join(difflib.ndiff(desired, actual)) diff --git a/test/test_pep8.py b/test/test_pep8.py index 81419e9..44c11f4 100644 --- a/test/test_pep8.py +++ b/test/test_pep8.py @@ -4,7 +4,7 @@ import parso def issues(code): grammar = parso.load_grammar() module = parso.parse(code) - return module._get_normalizer_issues(grammar) + return grammar._get_normalizer_issues(module) def test_eof_newline(): diff --git a/test/test_python_errors.py b/test/test_python_errors.py index 86457b0..026b639 100644 --- a/test/test_python_errors.py +++ b/test/test_python_errors.py @@ -13,7 +13,7 @@ from .failing_examples import FAILING_EXAMPLES, indent, build_nested def _get_error_list(code, version=None): grammar = parso.load_grammar(version=version) tree = grammar.parse(code) - return list(tree._iter_errors(grammar)) + return list(grammar.iter_errors(tree)) def assert_comparison(code, error_code, positions): errors = [(error.start_pos, error.code) for error in _get_error_list(code)]