VimPlug/parso
1
0
Fork 0
mirror of https://github.com/davidhalter/parso.git synced 2025-12-06 04:44:29 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Add NodeOrLeaf.dump() and NodeOrLeaf.search_ancestor() (#187)

Browse Source 
- Add `NodeOrLeaf.dump()` to generate a readable and "round-trippable" dump for a parser tree
- `parso.tree.search_ancestor()` is deprecated, use `NodeOrLeaf.search_ancestor()` instead
- Set up children's parent in `BaseNode.__init__()`
- Add test for `search_ancestor`
- Various small type annotations improvements
This commit is contained in:
Saiyang Gou
2021-05-29 12:40:07 -07:00
committed by GitHub
parent f2b1ff9429
commit 86f3f1096b
11 changed files with 364 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.coveragerc
View File

@@ -4,6 +4,8 @@ source = parso
[report]
# Regexes for lines to exclude from consideration
exclude_lines =
pragma: no cover
# Don't complain about missing debug-only code:
def __repr__

1
.gitignore vendored
View File

@@ -11,3 +11,4 @@ parso.egg-info/
/.pytest_cache
test/fuzz-redo.pickle
/venv/
/htmlcov/

8
parso/parser.py
View File

@@ -23,7 +23,7 @@ within the statement. This lowers memory usage and cpu time and reduces the
complexity of the ``Parser`` (there's another parser sitting inside
``Statement``, which produces ``Array`` and ``Call``).
"""
from typing import Dict
from typing import Dict, Type
from parso import tree
from parso.pgen2.generator import ReservedString
@@ -110,10 +110,10 @@ class BaseParser:
When a syntax error occurs, error_recovery() is called.
"""
node_map: Dict[str, type] = {}
node_map: Dict[str, Type[tree.BaseNode]] = {}
default_node = tree.Node
leaf_map: Dict[str, type] = {}
leaf_map: Dict[str, Type[tree.Leaf]] = {}
default_leaf = tree.Leaf
def __init__(self, pgen_grammar, start_nonterminal='file_input', error_recovery=False):
@@ -156,8 +156,6 @@ class BaseParser:
node = self.node_map[nonterminal](children)
except KeyError:
node = self.default_node(nonterminal, children)
for c in children:
c.parent = node
return node
def convert_leaf(self, type_, value, prefix, start_pos):

5
parso/python/errors.py
View File

@@ -5,7 +5,6 @@ import re
from contextlib import contextmanager
from parso.normalizer import Normalizer, NormalizerConfig, Issue, Rule
from parso.python.tree import search_ancestor
from parso.python.tokenize import _get_token_collection
_BLOCK_STMTS = ('if_stmt', 'while_stmt', 'for_stmt', 'try_stmt', 'with_stmt')
@@ -231,7 +230,7 @@ def _any_fstring_error(version, node):
elif node.type == "fstring":
return True
else:
return search_ancestor(node, "fstring")
return node.search_ancestor("fstring")
class _Context:
@@ -1265,7 +1264,7 @@ class _NamedExprRule(_CheckAssignmentRule):
def search_all_comp_ancestors(node):
has_ancestors = False
while True:
node = search_ancestor(node, 'testlist_comp', 'dictorsetmaker')
node = node.search_ancestor('testlist_comp', 'dictorsetmaker')
if node is None:
break
for child in node.children:

4
parso/python/parser.py
View File

@@ -96,8 +96,6 @@ class Parser(BaseParser):
# prefixes. Just ignore them.
children = [children[0]] + children[2:-1]
node = self.default_node(nonterminal, children)
for c in children:
c.parent = node
return node
def convert_leaf(self, type, value, prefix, start_pos):
@@ -185,8 +183,6 @@ class Parser(BaseParser):
if all_nodes:
node = tree.PythonErrorNode(all_nodes)
for n in all_nodes:
n.parent = node
self.stack[start_index - 1].nodes.append(node)
self.stack[start_index:] = []

8
parso/python/pep8.py
View File

@@ -4,7 +4,7 @@ from typing import Tuple
from parso.python.errors import ErrorFinder, ErrorFinderConfig
from parso.normalizer import Rule
from parso.python.tree import search_ancestor, Flow, Scope
from parso.python.tree import Flow, Scope
_IMPORT_TYPES = ('import_name', 'import_from')
@@ -124,7 +124,7 @@ class BackslashNode(IndentationNode):
type = IndentationTypes.BACKSLASH
def __init__(self, config, parent_indentation, containing_leaf, spacing, parent=None):
expr_stmt = search_ancestor(containing_leaf, 'expr_stmt')
expr_stmt = containing_leaf.search_ancestor('expr_stmt')
if expr_stmt is not None:
equals = expr_stmt.children[-2]
@@ -724,11 +724,11 @@ class PEP8Normalizer(ErrorFinder):
def add_issue(self, node, code, message):
if self._previous_leaf is not None:
if search_ancestor(self._previous_leaf, 'error_node') is not None:
if self._previous_leaf.search_ancestor('error_node') is not None:
return
if self._previous_leaf.type == 'error_leaf':
return
if search_ancestor(node, 'error_node') is not None:
if node.search_ancestor('error_node') is not None:
return
if code in (901, 903):
# 901 and 903 are raised by the ErrorFinder.

8
parso/python/prefix.py
View File

@@ -40,6 +40,14 @@ class PrefixPart:
self.start_pos
)
def search_ancestor(self, *node_types):
node = self.parent
while node is not None:
if node.type in node_types:
return node
node = node.parent
return None
_comment = r'#[^\n\r\f]*'
_backslash = r'\\\r?\n'

23
parso/python/tree.py
View File

@@ -49,8 +49,7 @@ except ImportError:
from collections import Mapping
from typing import Tuple
from parso.tree import Node, BaseNode, Leaf, ErrorNode, ErrorLeaf, \
search_ancestor
from parso.tree import Node, BaseNode, Leaf, ErrorNode, ErrorLeaf
from parso.python.prefix import split_prefix
from parso.utils import split_lines
@@ -549,7 +548,11 @@ class Function(ClassOrFunc):
def __init__(self, children):
super().__init__(children)
parameters = self.children[2] # After `def foo`
parameters.children[1:-1] = _create_params(parameters, parameters.children[1:-1])
parameters_children = parameters.children[1:-1]
# If input parameters list already has Param objects, keep it as is;
# otherwise, convert it to a list of Param objects.
if not any(isinstance(child, Param) for child in parameters_children):
parameters.children[1:-1] = _create_params(parameters, parameters_children)
def _get_param_nodes(self):
return self.children[2].children
@@ -652,7 +655,11 @@ class Lambda(Function):
# We don't want to call the Function constructor, call its parent.
super(Function, self).__init__(children)
# Everything between `lambda` and the `:` operator is a parameter.
self.children[1:-2] = _create_params(self, self.children[1:-2])
parameters_children = self.children[1:-2]
# If input children list already has Param objects, keep it as is;
# otherwise, convert it to a list of Param objects.
if not any(isinstance(child, Param) for child in parameters_children):
self.children[1:-2] = _create_params(self, parameters_children)
@property
def name(self):
@@ -776,7 +783,7 @@ class WithStmt(Flow):
return names
def get_test_node_from_name(self, name):
node = search_ancestor(name, "with_item")
node = name.search_ancestor("with_item")
if node is None:
raise ValueError('The name is not actually part of a with statement.')
return node.children[0]
@@ -1080,11 +1087,9 @@ class Param(PythonBaseNode):
"""
type = 'param'
def __init__(self, children, parent):
def __init__(self, children, parent=None):
super().__init__(children)
self.parent = parent
for child in children:
child.parent = self
@property
def star_count(self):
@@ -1171,7 +1176,7 @@ class Param(PythonBaseNode):
"""
Returns the function/lambda of a parameter.
"""
return search_ancestor(self, 'funcdef', 'lambdef')
return self.search_ancestor('funcdef', 'lambdef')
def get_code(self, include_prefix=True, include_comma=True):
"""

132
parso/tree.py
View File

@@ -1,22 +1,25 @@
from abc import abstractmethod, abstractproperty
from typing import List, Optional, Tuple
from typing import List, Optional, Tuple, Union
from parso.utils import split_lines
def search_ancestor(node, *node_types):
def search_ancestor(node: 'NodeOrLeaf', *node_types: str) -> 'Optional[BaseNode]':
"""
Recursively looks at the parents of a node and returns the first found node
that matches node_types. Returns ``None`` if no matching node is found.
that matches ``node_types``. Returns ``None`` if no matching node is found.
This function is deprecated, use :meth:`NodeOrLeaf.search_ancestor` instead.
:param node: The ancestors of this node will be checked.
:param node_types: type names that are searched for.
:type node_types: tuple of str
"""
while True:
node = node.parent
if node is None or node.type in node_types:
return node
n = node.parent
while n is not None:
if n.type in node_types:
return n
n = n.parent
return None
class NodeOrLeaf:
@@ -28,6 +31,11 @@ class NodeOrLeaf:
'''
The type is a string that typically matches the types of the grammar file.
'''
parent: 'Optional[BaseNode]'
'''
The parent :class:`BaseNode` of this node or leaf.
None if this is the root node.
'''
def get_root_node(self):
"""
@@ -173,6 +181,109 @@ class NodeOrLeaf:
e.g. a statement.
"""
def search_ancestor(self, *node_types: str) -> 'Optional[BaseNode]':
"""
Recursively looks at the parents of this node or leaf and returns the
first found node that matches ``node_types``. Returns ``None`` if no
matching node is found.
:param node_types: type names that are searched for.
"""
node = self.parent
while node is not None:
if node.type in node_types:
return node
node = node.parent
return None
def dump(self, *, indent: Optional[Union[int, str]] = 4) -> str:
"""
Returns a formatted dump of the parser tree rooted at this node or leaf. This is
mainly useful for debugging purposes.
The ``indent`` parameter is interpreted in a similar way as :py:func:`ast.dump`.
If ``indent`` is a non-negative integer or string, then the tree will be
pretty-printed with that indent level. An indent level of 0, negative, or ``""``
will only insert newlines. ``None`` selects the single line representation.
Using a positive integer indent indents that many spaces per level. If
``indent`` is a string (such as ``"\\t"``), that string is used to indent each
level.
:param indent: Indentation style as described above. The default indentation is
4 spaces, which yields a pretty-printed dump.
>>> import parso
>>> print(parso.parse("lambda x, y: x + y").dump())
Module([
Lambda([
Keyword('lambda', (1, 0)),
Param([
Name('x', (1, 7), prefix=' '),
Operator(',', (1, 8)),
]),
Param([
Name('y', (1, 10), prefix=' '),
]),
Operator(':', (1, 11)),
PythonNode('arith_expr', [
Name('x', (1, 13), prefix=' '),
Operator('+', (1, 15), prefix=' '),
Name('y', (1, 17), prefix=' '),
]),
]),
EndMarker('', (1, 18)),
])
"""
if indent is None:
newline = False
indent_string = ''
elif isinstance(indent, int):
newline = True
indent_string = ' ' * indent
elif isinstance(indent, str):
newline = True
indent_string = indent
else:
raise TypeError(f"expect 'indent' to be int, str or None, got {indent!r}")
def _format_dump(node: NodeOrLeaf, indent: str = '', top_level: bool = True) -> str:
result = ''
node_type = type(node).__name__
if isinstance(node, Leaf):
result += f'{indent}{node_type}('
if isinstance(node, ErrorLeaf):
result += f'{node.token_type!r}, '
elif isinstance(node, TypedLeaf):
result += f'{node.type!r}, '
result += f'{node.value!r}, {node.start_pos!r}'
if node.prefix:
result += f', prefix={node.prefix!r}'
result += ')'
elif isinstance(node, BaseNode):
result += f'{indent}{node_type}('
if isinstance(node, Node):
result += f'{node.type!r}, '
result += '['
if newline:
result += '\n'
for child in node.children:
result += _format_dump(child, indent=indent + indent_string, top_level=False)
result += f'{indent}])'
else: # pragma: no cover
# We shouldn't ever reach here, unless:
# - `NodeOrLeaf` is incorrectly subclassed else where
# - or a node's children list contains invalid nodes or leafs
# Both are unexpected internal errors.
raise TypeError(f'unsupported node encountered: {node!r}')
if not top_level:
if newline:
result += ',\n'
else:
result += ', '
return result
return _format_dump(self)
class Leaf(NodeOrLeaf):
'''
@@ -180,6 +291,7 @@ class Leaf(NodeOrLeaf):
were defined and what text preceeds them.
'''
__slots__ = ('value', 'parent', 'line', 'column', 'prefix')
prefix: str
def __init__(self, value: str, start_pos: Tuple[int, int], prefix: str = '') -> None:
self.value = value
@@ -266,9 +378,11 @@ class BaseNode(NodeOrLeaf):
"""
self.parent: Optional[BaseNode] = None
'''
The parent :class:`BaseNode` of this leaf.
The parent :class:`BaseNode` of this node.
None if this is the root node.
'''
for child in children:
child.parent = self
@property
def start_pos(self) -> Tuple[int, int]:

182
test/test_dump_tree.py Normal file
View File

@@ -0,0 +1,182 @@
from textwrap import dedent
import pytest
from parso import parse
# Using star import for easier eval testing below.
from parso.python.tree import * # noqa: F403
from parso.tree import * # noqa: F403
from parso.tree import ErrorLeaf, TypedLeaf
@pytest.mark.parametrize(
'indent,expected_dump', [
(None, "Module(["
"Lambda(["
"Keyword('lambda', (1, 0)), "
"Param(["
"Name('x', (1, 7), prefix=' '), "
"Operator(',', (1, 8)), "
"]), "
"Param(["
"Name('y', (1, 10), prefix=' '), "
"]), "
"Operator(':', (1, 11)), "
"PythonNode('arith_expr', ["
"Name('x', (1, 13), prefix=' '), "
"Operator('+', (1, 15), prefix=' '), "
"Name('y', (1, 17), prefix=' '), "
"]), "
"]), "
"EndMarker('', (1, 18)), "
"])"),
(0, dedent('''\
Module([
Lambda([
Keyword('lambda', (1, 0)),
Param([
Name('x', (1, 7), prefix=' '),
Operator(',', (1, 8)),
]),
Param([
Name('y', (1, 10), prefix=' '),
]),
Operator(':', (1, 11)),
PythonNode('arith_expr', [
Name('x', (1, 13), prefix=' '),
Operator('+', (1, 15), prefix=' '),
Name('y', (1, 17), prefix=' '),
]),
]),
EndMarker('', (1, 18)),
])''')),
(4, dedent('''\
Module([
Lambda([
Keyword('lambda', (1, 0)),
Param([
Name('x', (1, 7), prefix=' '),
Operator(',', (1, 8)),
]),
Param([
Name('y', (1, 10), prefix=' '),
]),
Operator(':', (1, 11)),
PythonNode('arith_expr', [
Name('x', (1, 13), prefix=' '),
Operator('+', (1, 15), prefix=' '),
Name('y', (1, 17), prefix=' '),
]),
]),
EndMarker('', (1, 18)),
])''')),
('\t', dedent('''\
Module([
\tLambda([
\t\tKeyword('lambda', (1, 0)),
\t\tParam([
\t\t\tName('x', (1, 7), prefix=' '),
\t\t\tOperator(',', (1, 8)),
\t\t]),
\t\tParam([
\t\t\tName('y', (1, 10), prefix=' '),
\t\t]),
\t\tOperator(':', (1, 11)),
\t\tPythonNode('arith_expr', [
\t\t\tName('x', (1, 13), prefix=' '),
\t\t\tOperator('+', (1, 15), prefix=' '),
\t\t\tName('y', (1, 17), prefix=' '),
\t\t]),
\t]),
\tEndMarker('', (1, 18)),
])''')),
]
)
def test_dump_parser_tree(indent, expected_dump):
code = "lambda x, y: x + y"
module = parse(code)
assert module.dump(indent=indent) == expected_dump
# Check that dumped tree can be eval'd to recover the parser tree and original code.
recovered_code = eval(expected_dump).get_code()
assert recovered_code == code
@pytest.mark.parametrize(
'node,expected_dump,expected_code', [
( # Dump intermediate node (not top level module)
parse("def foo(x, y): return x + y").children[0], dedent('''\
Function([
Keyword('def', (1, 0)),
Name('foo', (1, 4), prefix=' '),
PythonNode('parameters', [
Operator('(', (1, 7)),
Param([
Name('x', (1, 8)),
Operator(',', (1, 9)),
]),
Param([
Name('y', (1, 11), prefix=' '),
]),
Operator(')', (1, 12)),
]),
Operator(':', (1, 13)),
ReturnStmt([
Keyword('return', (1, 15), prefix=' '),
PythonNode('arith_expr', [
Name('x', (1, 22), prefix=' '),
Operator('+', (1, 24), prefix=' '),
Name('y', (1, 26), prefix=' '),
]),
]),
])'''),
"def foo(x, y): return x + y",
),
( # Dump leaf
parse("def foo(x, y): return x + y").children[0].children[0],
"Keyword('def', (1, 0))",
'def',
),
( # Dump ErrorLeaf
ErrorLeaf('error_type', 'error_code', (1, 1), prefix=' '),
"ErrorLeaf('error_type', 'error_code', (1, 1), prefix=' ')",
' error_code',
),
( # Dump TypedLeaf
TypedLeaf('type', 'value', (1, 1)),
"TypedLeaf('type', 'value', (1, 1))",
'value',
),
]
)
def test_dump_parser_tree_not_top_level_module(node, expected_dump, expected_code):
dump_result = node.dump()
assert dump_result == expected_dump
# Check that dumped tree can be eval'd to recover the parser tree and original code.
recovered_code = eval(dump_result).get_code()
assert recovered_code == expected_code
def test_dump_parser_tree_invalid_args():
module = parse("lambda x, y: x + y")
with pytest.raises(TypeError):
module.dump(indent=1.1)
def test_eval_dump_recovers_parent():
module = parse("lambda x, y: x + y")
module2 = eval(module.dump())
assert module2.parent is None
lambda_node = module2.children[0]
assert lambda_node.parent is module2
assert module2.children[1].parent is module2
assert lambda_node.children[0].parent is lambda_node
param_node = lambda_node.children[1]
assert param_node.parent is lambda_node
assert param_node.children[0].parent is param_node
assert param_node.children[1].parent is param_node
arith_expr_node = lambda_node.children[-1]
assert arith_expr_node.parent is lambda_node
assert arith_expr_node.children[0].parent is arith_expr_node

25
test/test_parser_tree.py
View File

@@ -6,6 +6,7 @@ import pytest
from parso import parse
from parso.python import tree
from parso.tree import search_ancestor
class TestsFunctionAndLambdaParsing:
@@ -239,3 +240,27 @@ def test_with_stmt_get_test_node_from_name():
for name in with_stmt.get_defined_names(include_setitem=True)
]
assert tests == ["A", "B", "C", "D"]
sample_module = parse('x + y')
sample_node = sample_module.children[0]
sample_leaf = sample_node.children[0]
@pytest.mark.parametrize(
'node,node_types,expected_ancestor', [
(sample_module, ('file_input',), None),
(sample_node, ('arith_expr',), None),
(sample_node, ('file_input', 'eval_input'), sample_module),
(sample_leaf, ('name',), None),
(sample_leaf, ('arith_expr',), sample_node),
(sample_leaf, ('file_input',), sample_module),
(sample_leaf, ('file_input', 'arith_expr'), sample_node),
(sample_leaf, ('shift_expr',), None),
(sample_leaf, ('name', 'shift_expr',), None),
(sample_leaf, (), None),
]
)
def test_search_ancestor(node, node_types, expected_ancestor):
assert node.search_ancestor(*node_types) is expected_ancestor
assert search_ancestor(node, *node_types) is expected_ancestor # deprecated