mirror of
https://github.com/davidhalter/parso.git
synced 2025-12-08 21:54:54 +08:00
Compare commits
20 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
5a57e8df06 | ||
|
|
aa82a1d39a | ||
|
|
e0f74dd8ad | ||
|
|
7df254440e | ||
|
|
05a8236d3f | ||
|
|
2067845cef | ||
|
|
dcdd3bbc8e | ||
|
|
82868580a2 | ||
|
|
a18baf0d2c | ||
|
|
ed803f5749 | ||
|
|
032c7563c4 | ||
|
|
b83c641057 | ||
|
|
97d9aeafb7 | ||
|
|
7a277c7302 | ||
|
|
5993765e0a | ||
|
|
435d310c2b | ||
|
|
8aa280342a | ||
|
|
73c61bca4a | ||
|
|
60ed141d80 | ||
|
|
091e72562c |
63
README.rst
63
README.rst
@@ -1,5 +1,5 @@
|
|||||||
###################################################################
|
###################################################################
|
||||||
parso - A Python Parser Written in Python
|
parso - A Python Parser
|
||||||
###################################################################
|
###################################################################
|
||||||
|
|
||||||
.. image:: https://secure.travis-ci.org/davidhalter/parso.png?branch=master
|
.. image:: https://secure.travis-ci.org/davidhalter/parso.png?branch=master
|
||||||
@@ -10,19 +10,52 @@ parso - A Python Parser Written in Python
|
|||||||
:target: https://coveralls.io/r/davidhalter/parso
|
:target: https://coveralls.io/r/davidhalter/parso
|
||||||
:alt: Coverage Status
|
:alt: Coverage Status
|
||||||
|
|
||||||
|
.. image:: https://raw.githubusercontent.com/davidhalter/parso/master/docs/_static/logo_characters.png
|
||||||
|
|
||||||
Parso is a Python parser that supports error recovery and round-trip parsing.
|
Parso is a Python parser that supports error recovery and round-trip parsing
|
||||||
|
for different Python versions (in multiple Python versions). Parso is also able
|
||||||
|
to list multiple syntax errors in your python file.
|
||||||
|
|
||||||
Parso has been battle-tested by jedi_. It was pulled out of jedi to be useful
|
Parso has been battle-tested by jedi_. It was pulled out of jedi to be useful
|
||||||
for other projects as well.
|
for other projects as well.
|
||||||
|
|
||||||
Parso is very simplistic. It consists of a small API to parse Python and
|
Parso consists of a small API to parse Python and analyse the syntax tree.
|
||||||
analyse the parsing tree.
|
|
||||||
|
|
||||||
|
A simple example:
|
||||||
|
|
||||||
|
.. code-block:: python
|
||||||
|
|
||||||
|
>>> import parso
|
||||||
|
>>> module = parso.parse('hello + 1', version="3.6")
|
||||||
|
>>> expr = module.children[0]
|
||||||
|
>>> expr
|
||||||
|
PythonNode(arith_expr, [<Name: hello@1,0>, <Operator: +>, <Number: 1>])
|
||||||
|
>>> print(expr.get_code())
|
||||||
|
hello + 1
|
||||||
|
>>> name = expr.children[0]
|
||||||
|
>>> name
|
||||||
|
<Name: hello@1,0>
|
||||||
|
>>> name.end_pos
|
||||||
|
(1, 5)
|
||||||
|
>>> expr.end_pos
|
||||||
|
(1, 9)
|
||||||
|
|
||||||
|
To list multiple issues:
|
||||||
|
|
||||||
|
.. code-block:: python
|
||||||
|
|
||||||
|
>>> grammar = parso.load_grammar()
|
||||||
|
>>> module = grammar.parse('foo +\nbar\ncontinue')
|
||||||
|
>>> error1, error2 = grammar.iter_errors(module)
|
||||||
|
>>> error1.message
|
||||||
|
'SyntaxError: invalid syntax'
|
||||||
|
>>> error2.message
|
||||||
|
"SyntaxError: 'continue' not properly in loop"
|
||||||
|
|
||||||
Ressources
|
Ressources
|
||||||
==========
|
==========
|
||||||
|
|
||||||
|
- `Testing <http://parso.readthedocs.io/en/latest/docs/development.html#testing>`_
|
||||||
- `PyPI <https://pypi.python.org/pypi/parso>`_
|
- `PyPI <https://pypi.python.org/pypi/parso>`_
|
||||||
- `Docs <https://parso.readthedocs.org/en/latest/>`_
|
- `Docs <https://parso.readthedocs.org/en/latest/>`_
|
||||||
- Uses `semantic versioning <http://semver.org/>`_
|
- Uses `semantic versioning <http://semver.org/>`_
|
||||||
@@ -42,32 +75,16 @@ Known Issues
|
|||||||
============
|
============
|
||||||
|
|
||||||
- `async`/`await` are already used as keywords in Python3.6.
|
- `async`/`await` are already used as keywords in Python3.6.
|
||||||
- `from __future__ import print_function` is not supported,
|
- `from __future__ import print_function` is not ignored.
|
||||||
|
|
||||||
Testing
|
|
||||||
=======
|
|
||||||
|
|
||||||
The test suite depends on ``tox`` and ``pytest``::
|
|
||||||
|
|
||||||
pip install tox pytest
|
|
||||||
|
|
||||||
To run the tests for all supported Python versions::
|
|
||||||
|
|
||||||
tox
|
|
||||||
|
|
||||||
If you want to test only a specific Python version (e.g. Python 2.7), it's as
|
|
||||||
easy as ::
|
|
||||||
|
|
||||||
tox -e py27
|
|
||||||
|
|
||||||
Tests are also run automatically on `Travis CI
|
|
||||||
<https://travis-ci.org/davidhalter/parso/>`_.
|
|
||||||
|
|
||||||
Acknowledgements
|
Acknowledgements
|
||||||
================
|
================
|
||||||
|
|
||||||
- Guido van Rossum (@gvanrossum) for creating the parser generator pgen2
|
- Guido van Rossum (@gvanrossum) for creating the parser generator pgen2
|
||||||
(originally used in lib2to3).
|
(originally used in lib2to3).
|
||||||
|
- `Salome Schneider <https://www.crepes-schnaegg.ch/cr%C3%AApes-schn%C3%A4gg/kunst-f%C3%BCrs-cr%C3%AApes-mobil/>`_
|
||||||
|
for the extremely awesome parso logo.
|
||||||
|
|
||||||
|
|
||||||
.. _jedi: https://github.com/davidhalter/jedi
|
.. _jedi: https://github.com/davidhalter/jedi
|
||||||
|
|||||||
BIN
docs/_static/logo.png
vendored
BIN
docs/_static/logo.png
vendored
Binary file not shown.
|
Before Width: | Height: | Size: 28 KiB After Width: | Height: | Size: 200 KiB |
BIN
docs/_static/logo_characters.png
vendored
Normal file
BIN
docs/_static/logo_characters.png
vendored
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 55 KiB |
2
docs/_themes/flask/layout.html
vendored
2
docs/_themes/flask/layout.html
vendored
@@ -7,7 +7,7 @@
|
|||||||
<link media="only screen and (max-device-width: 480px)" href="{{
|
<link media="only screen and (max-device-width: 480px)" href="{{
|
||||||
pathto('_static/small_flask.css', 1) }}" type= "text/css" rel="stylesheet" />
|
pathto('_static/small_flask.css', 1) }}" type= "text/css" rel="stylesheet" />
|
||||||
<a href="https://github.com/davidhalter/jedi">
|
<a href="https://github.com/davidhalter/jedi">
|
||||||
<img style="position: absolute; top: 0; right: 0; border: 0;" src="https://s3.amazonaws.com/github/ribbons/forkme_right_red_aa0000.png" alt="Fork me on GitHub">
|
<img style="position: absolute; top: 0; right: 0; border: 0;" src="https://s3.amazonaws.com/github/ribbons/forkme_right_red_aa0000.png" alt="Fork me">
|
||||||
</a>
|
</a>
|
||||||
{% endblock %}
|
{% endblock %}
|
||||||
{%- block relbar2 %}{% endblock %}
|
{%- block relbar2 %}{% endblock %}
|
||||||
|
|||||||
@@ -274,7 +274,7 @@ autodoc_default_flags = []
|
|||||||
# -- Options for intersphinx module --------------------------------------------
|
# -- Options for intersphinx module --------------------------------------------
|
||||||
|
|
||||||
intersphinx_mapping = {
|
intersphinx_mapping = {
|
||||||
'http://docs.python.org/': None,
|
'http://docs.python.org/': ('https://docs.python.org/3.6', None),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -4,7 +4,8 @@ Development
|
|||||||
===========
|
===========
|
||||||
|
|
||||||
If you want to contribute anything to |parso|, just open an issue or pull
|
If you want to contribute anything to |parso|, just open an issue or pull
|
||||||
request to discuss it. We welcome changes!
|
request to discuss it. We welcome changes! Please check the ``CONTRIBUTING.md``
|
||||||
|
file in the repository, first.
|
||||||
|
|
||||||
|
|
||||||
Deprecations Process
|
Deprecations Process
|
||||||
|
|||||||
@@ -3,8 +3,8 @@
|
|||||||
Installation and Configuration
|
Installation and Configuration
|
||||||
==============================
|
==============================
|
||||||
|
|
||||||
The preferred way
|
The preferred way (pip)
|
||||||
-----------------
|
-----------------------
|
||||||
|
|
||||||
On any system you can install |parso| directly from the Python package index
|
On any system you can install |parso| directly from the Python package index
|
||||||
using pip::
|
using pip::
|
||||||
|
|||||||
@@ -1,29 +1,42 @@
|
|||||||
|
.. include:: ../global.rst
|
||||||
|
|
||||||
.. _parser-tree:
|
.. _parser-tree:
|
||||||
|
|
||||||
Parser Tree
|
Parser Tree
|
||||||
===========
|
===========
|
||||||
|
|
||||||
Usage
|
The parser tree is returned by calling :py:meth:`parso.Grammar.parse`.
|
||||||
-----
|
|
||||||
|
|
||||||
.. automodule:: parso.python
|
.. note:: Note that parso positions are always 1 based for lines and zero
|
||||||
|
based for columns. This means the first position in a file is (1, 0).
|
||||||
|
|
||||||
|
Parser Tree Base Classes
|
||||||
|
------------------------
|
||||||
|
|
||||||
|
Generally there are two types of classes you will deal with:
|
||||||
|
:py:class:`parso.tree.Leaf` and :py:class:`parso.tree.BaseNode`.
|
||||||
|
|
||||||
|
.. autoclass:: parso.tree.BaseNode
|
||||||
|
:show-inheritance:
|
||||||
:members:
|
:members:
|
||||||
:undoc-members:
|
|
||||||
|
|
||||||
|
.. autoclass:: parso.tree.Leaf
|
||||||
Parser Tree Base Class
|
:show-inheritance:
|
||||||
----------------------
|
:members:
|
||||||
|
|
||||||
All nodes and leaves have these methods/properties:
|
All nodes and leaves have these methods/properties:
|
||||||
|
|
||||||
.. autoclass:: parso.tree.NodeOrLeaf
|
.. autoclass:: parso.tree.NodeOrLeaf
|
||||||
:members:
|
:members:
|
||||||
:undoc-members:
|
:undoc-members:
|
||||||
|
:show-inheritance:
|
||||||
|
|
||||||
|
|
||||||
Python Parser Tree
|
Python Parser Tree
|
||||||
------------------
|
------------------
|
||||||
|
|
||||||
|
.. currentmodule:: parso.python.tree
|
||||||
|
|
||||||
.. automodule:: parso.python.tree
|
.. automodule:: parso.python.tree
|
||||||
:members:
|
:members:
|
||||||
:undoc-members:
|
:undoc-members:
|
||||||
|
|||||||
@@ -4,12 +4,13 @@ Usage
|
|||||||
=====
|
=====
|
||||||
|
|
||||||
|parso| works around grammars. You can simply create Python grammars by calling
|
|parso| works around grammars. You can simply create Python grammars by calling
|
||||||
``load_grammar``. Grammars (with a custom tokenizer and custom parser trees)
|
:py:func:`parso.load_grammar`. Grammars (with a custom tokenizer and custom parser trees)
|
||||||
can also be created by directly instantiating ``Grammar``. More information
|
can also be created by directly instantiating :py:func:`parso.Grammar`. More information
|
||||||
about the resulting objects can be found in the :ref:`parser tree documentation
|
about the resulting objects can be found in the :ref:`parser tree documentation
|
||||||
<parser-tree>`.
|
<parser-tree>`.
|
||||||
|
|
||||||
The simplest way of using parso is without even loading a grammar:
|
The simplest way of using parso is without even loading a grammar
|
||||||
|
(:py:func:`parso.parse`):
|
||||||
|
|
||||||
.. sourcecode:: python
|
.. sourcecode:: python
|
||||||
|
|
||||||
@@ -17,7 +18,31 @@ The simplest way of using parso is without even loading a grammar:
|
|||||||
>>> parso.parse('foo + bar')
|
>>> parso.parse('foo + bar')
|
||||||
<Module: @1-1>
|
<Module: @1-1>
|
||||||
|
|
||||||
.. automodule:: parso.grammar
|
Loading a Grammar
|
||||||
|
-----------------
|
||||||
|
|
||||||
|
Typically if you want to work with one specific Python version, use:
|
||||||
|
|
||||||
|
.. autofunction:: parso.load_grammar
|
||||||
|
|
||||||
|
Grammar methods
|
||||||
|
---------------
|
||||||
|
|
||||||
|
You will get back a grammar object that you can use to parse code and find
|
||||||
|
issues in it:
|
||||||
|
|
||||||
|
.. autoclass:: parso.Grammar
|
||||||
|
:members:
|
||||||
|
:undoc-members:
|
||||||
|
|
||||||
|
|
||||||
|
Error Retrieval
|
||||||
|
---------------
|
||||||
|
|
||||||
|
|parso| is able to find multiple errors in your source code. Iterating through
|
||||||
|
those errors yields the following instances:
|
||||||
|
|
||||||
|
.. autoclass:: parso.normalizer.Issue
|
||||||
:members:
|
:members:
|
||||||
:undoc-members:
|
:undoc-members:
|
||||||
|
|
||||||
@@ -25,17 +50,17 @@ The simplest way of using parso is without even loading a grammar:
|
|||||||
Utility
|
Utility
|
||||||
-------
|
-------
|
||||||
|
|
||||||
.. autofunction:: parso.parse
|
|parso| also offers some utility functions that can be really useful:
|
||||||
|
|
||||||
.. automodule:: parso.utils
|
.. autofunction:: parso.parse
|
||||||
:members:
|
.. autofunction:: parso.split_lines
|
||||||
:undoc-members:
|
.. autofunction:: parso.python_bytes_to_unicode
|
||||||
|
|
||||||
|
|
||||||
Used By
|
Used By
|
||||||
-------
|
-------
|
||||||
|
|
||||||
- jedi_ (which is used by IPython and a lot of plugins).
|
- jedi_ (which is used by IPython and a lot of editor plugins).
|
||||||
|
|
||||||
|
|
||||||
.. _jedi: https://github.com/davidhalter/jedi
|
.. _jedi: https://github.com/davidhalter/jedi
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
.. include global.rst
|
.. include global.rst
|
||||||
|
|
||||||
parso - A Python Parser Written in Python
|
parso - A Python Parser
|
||||||
=========================================
|
=======================
|
||||||
|
|
||||||
Release v\ |release|. (:doc:`Installation <docs/installation>`)
|
Release v\ |release|. (:doc:`Installation <docs/installation>`)
|
||||||
|
|
||||||
|
|||||||
@@ -1,10 +1,19 @@
|
|||||||
"""
|
r"""
|
||||||
parso is a Python parser. It's really easy to use and supports multiple Python
|
Parso is a Python parser that supports error recovery and round-trip parsing
|
||||||
versions, file caching, round-trips and other stuff:
|
for different Python versions (in multiple Python versions). Parso is also able
|
||||||
|
to list multiple syntax errors in your python file.
|
||||||
|
|
||||||
>>> from parso import load_grammar
|
Parso has been battle-tested by jedi_. It was pulled out of jedi to be useful
|
||||||
>>> grammar = load_grammar(version='2.7')
|
for other projects as well.
|
||||||
>>> module = grammar.parse('hello + 1')
|
|
||||||
|
Parso consists of a small API to parse Python and analyse the syntax tree.
|
||||||
|
|
||||||
|
.. _jedi: https://github.com/davidhalter/jedi
|
||||||
|
|
||||||
|
A simple example:
|
||||||
|
|
||||||
|
>>> import parso
|
||||||
|
>>> module = parso.parse('hello + 1', version="3.6")
|
||||||
>>> expr = module.children[0]
|
>>> expr = module.children[0]
|
||||||
>>> expr
|
>>> expr
|
||||||
PythonNode(arith_expr, [<Name: hello@1,0>, <Operator: +>, <Number: 1>])
|
PythonNode(arith_expr, [<Name: hello@1,0>, <Operator: +>, <Number: 1>])
|
||||||
@@ -17,19 +26,32 @@ hello + 1
|
|||||||
(1, 5)
|
(1, 5)
|
||||||
>>> expr.end_pos
|
>>> expr.end_pos
|
||||||
(1, 9)
|
(1, 9)
|
||||||
|
|
||||||
|
To list multiple issues:
|
||||||
|
|
||||||
|
>>> grammar = parso.load_grammar()
|
||||||
|
>>> module = grammar.parse('foo +\nbar\ncontinue')
|
||||||
|
>>> error1, error2 = grammar.iter_errors(module)
|
||||||
|
>>> error1.message
|
||||||
|
'SyntaxError: invalid syntax'
|
||||||
|
>>> error2.message
|
||||||
|
"SyntaxError: 'continue' not properly in loop"
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from parso.parser import ParserSyntaxError
|
from parso.parser import ParserSyntaxError
|
||||||
from parso.grammar import Grammar, load_grammar
|
from parso.grammar import Grammar, load_grammar
|
||||||
|
from parso.utils import split_lines, python_bytes_to_unicode
|
||||||
|
|
||||||
|
|
||||||
__version__ = '0.0.4'
|
__version__ = '0.1.0'
|
||||||
|
|
||||||
|
|
||||||
def parse(code=None, **kwargs):
|
def parse(code=None, **kwargs):
|
||||||
"""
|
"""
|
||||||
A utility function to parse Python with the current Python version. Params
|
A utility function to avoid loading grammars.
|
||||||
are documented in ``Grammar.parse``.
|
Params are documented in :py:meth:`parso.Grammar.parse`.
|
||||||
|
|
||||||
|
:param str version: The version used by :py:func:`parso.load_grammar`.
|
||||||
"""
|
"""
|
||||||
version = kwargs.pop('version', None)
|
version = kwargs.pop('version', None)
|
||||||
grammar = load_grammar(version=version)
|
grammar = load_grammar(version=version)
|
||||||
|
|||||||
@@ -19,10 +19,11 @@ _loaded_grammars = {}
|
|||||||
|
|
||||||
class Grammar(object):
|
class Grammar(object):
|
||||||
"""
|
"""
|
||||||
Create custom grammars by calling this. It's not really supported, yet.
|
:py:func:`parso.load_grammar` returns instances of this class.
|
||||||
|
|
||||||
:param text: A BNF representation of your grammar.
|
Creating custom grammars by calling this is not supported, yet.
|
||||||
"""
|
"""
|
||||||
|
#:param text: A BNF representation of your grammar.
|
||||||
_error_normalizer_config = None
|
_error_normalizer_config = None
|
||||||
_token_namespace = None
|
_token_namespace = None
|
||||||
_default_normalizer_config = pep8.PEP8NormalizerConfig()
|
_default_normalizer_config = pep8.PEP8NormalizerConfig()
|
||||||
@@ -44,33 +45,38 @@ class Grammar(object):
|
|||||||
If you need finer grained control over the parsed instance, there will be
|
If you need finer grained control over the parsed instance, there will be
|
||||||
other ways to access it.
|
other ways to access it.
|
||||||
|
|
||||||
:param code str: A unicode string that contains Python code.
|
:param str code: A unicode or bytes string. When it's not possible to
|
||||||
:param path str: The path to the file you want to open. Only needed for caching.
|
decode bytes to a string, returns a
|
||||||
:param error_recovery bool: If enabled, any code will be returned. If
|
:py:class:`UnicodeDecodeError`.
|
||||||
|
:param bool error_recovery: If enabled, any code will be returned. If
|
||||||
it is invalid, it will be returned as an error node. If disabled,
|
it is invalid, it will be returned as an error node. If disabled,
|
||||||
you will get a ParseError when encountering syntax errors in your
|
you will get a ParseError when encountering syntax errors in your
|
||||||
code.
|
code.
|
||||||
:param start_symbol str: The grammar symbol that you want to parse. Only
|
:param str start_symbol: The grammar symbol that you want to parse. Only
|
||||||
allowed to be used when error_recovery is False.
|
allowed to be used when error_recovery is False.
|
||||||
:param cache bool: Keeps a copy of the parser tree in RAM and on disk
|
:param str path: The path to the file you want to open. Only needed for caching.
|
||||||
|
:param bool cache: Keeps a copy of the parser tree in RAM and on disk
|
||||||
if a path is given. Returns the cached trees if the corresponding
|
if a path is given. Returns the cached trees if the corresponding
|
||||||
files on disk have not changed.
|
files on disk have not changed.
|
||||||
:param diff_cache bool: Diffs the cached python module against the new
|
:param bool diff_cache: Diffs the cached python module against the new
|
||||||
code and tries to parse only the parts that have changed. Returns
|
code and tries to parse only the parts that have changed. Returns
|
||||||
the same (changed) module that is found in cache. Using this option
|
the same (changed) module that is found in cache. Using this option
|
||||||
requires you to not do anything anymore with the old cached module,
|
requires you to not do anything anymore with the cached modules
|
||||||
because the contents of it might have changed.
|
under that path, because the contents of it might change. This
|
||||||
:param cache_path bool: If given saves the parso cache in this
|
option is still somewhat experimental. If you want stability,
|
||||||
|
please don't use it.
|
||||||
|
:param bool cache_path: If given saves the parso cache in this
|
||||||
directory. If not given, defaults to the default cache places on
|
directory. If not given, defaults to the default cache places on
|
||||||
each platform.
|
each platform.
|
||||||
|
|
||||||
:return: A syntax tree node. Typically the module.
|
:return: A subclass of :py:class:`parso.tree.NodeOrLeaf`. Typically a
|
||||||
|
:py:class:`parso.python.tree.Module`.
|
||||||
"""
|
"""
|
||||||
if 'start_pos' in kwargs:
|
if 'start_pos' in kwargs:
|
||||||
raise TypeError("parse() got an unexpected keyworda argument.")
|
raise TypeError("parse() got an unexpected keyworda argument.")
|
||||||
return self._parse(code=code, **kwargs)
|
return self._parse(code=code, **kwargs)
|
||||||
|
|
||||||
def _parse(self, code=None, path=None, error_recovery=True,
|
def _parse(self, code=None, error_recovery=True, path=None,
|
||||||
start_symbol=None, cache=False, diff_cache=False,
|
start_symbol=None, cache=False, diff_cache=False,
|
||||||
cache_path=None, start_pos=(1, 0)):
|
cache_path=None, start_pos=(1, 0)):
|
||||||
"""
|
"""
|
||||||
@@ -152,6 +158,11 @@ class Grammar(object):
|
|||||||
return ns
|
return ns
|
||||||
|
|
||||||
def iter_errors(self, node):
|
def iter_errors(self, node):
|
||||||
|
"""
|
||||||
|
Given a :py:class:`parso.tree.NodeOrLeaf` returns a generator of
|
||||||
|
:py:class:`parso.normalizer.Issue` objects. For Python this is
|
||||||
|
a list of syntax/indentation errors.
|
||||||
|
"""
|
||||||
if self._error_normalizer_config is None:
|
if self._error_normalizer_config is None:
|
||||||
raise ValueError("No error normalizer specified for this grammar.")
|
raise ValueError("No error normalizer specified for this grammar.")
|
||||||
|
|
||||||
@@ -237,10 +248,10 @@ class PythonFStringGrammar(Grammar):
|
|||||||
|
|
||||||
def load_grammar(**kwargs):
|
def load_grammar(**kwargs):
|
||||||
"""
|
"""
|
||||||
Loads a Python grammar. The default version is the current Python version.
|
Loads a :py:class:`parso.Grammar`. The default version is the current Python
|
||||||
|
version.
|
||||||
|
|
||||||
If you need support for a specific version, please use e.g.
|
:param str version: A python version string, e.g. ``version='3.3'``.
|
||||||
`version='3.3'`.
|
|
||||||
"""
|
"""
|
||||||
def load_grammar(language='python', version=None):
|
def load_grammar(language='python', version=None):
|
||||||
if language == 'python':
|
if language == 'python':
|
||||||
|
|||||||
@@ -121,8 +121,18 @@ class Issue(object):
|
|||||||
def __init__(self, node, code, message):
|
def __init__(self, node, code, message):
|
||||||
self._node = node
|
self._node = node
|
||||||
self.code = code
|
self.code = code
|
||||||
|
"""
|
||||||
|
An integer code that stands for the type of error.
|
||||||
|
"""
|
||||||
self.message = message
|
self.message = message
|
||||||
|
"""
|
||||||
|
A message (string) for the issue.
|
||||||
|
"""
|
||||||
self.start_pos = node.start_pos
|
self.start_pos = node.start_pos
|
||||||
|
"""
|
||||||
|
The start position position of the error as a tuple (line, column). As
|
||||||
|
always in |parso| the first line is 1 and the first column 0.
|
||||||
|
"""
|
||||||
|
|
||||||
def __eq__(self, other):
|
def __eq__(self, other):
|
||||||
return self.start_pos == other.start_pos and self.code == other.code
|
return self.start_pos == other.start_pos and self.code == other.code
|
||||||
|
|||||||
@@ -1,16 +1,18 @@
|
|||||||
"""
|
"""
|
||||||
If you know what an syntax tree is, you'll see that this module is pretty much
|
This is the syntax tree for Python syntaxes (2 & 3). The classes represent
|
||||||
that. The classes represent syntax elements like functions and imports.
|
syntax elements like functions and imports.
|
||||||
|
|
||||||
This is the "business logic" part of the parser. There's a lot of logic here
|
All of the nodes can be traced back to the `Python grammar file
|
||||||
that makes it easier for Jedi (and other libraries) to deal with a Python syntax
|
<https://docs.python.org/3/reference/grammar.html>`_. If you want to know how
|
||||||
tree.
|
a tree is structured, just analyse that file (for each Python version it's a
|
||||||
|
bit different).
|
||||||
|
|
||||||
By using `get_code` on a module, you can get back the 1-to-1 representation of
|
There's a lot of logic here that makes it easier for Jedi (and other libraries)
|
||||||
the input given to the parser. This is important if you are using refactoring.
|
to deal with a Python syntax tree.
|
||||||
|
|
||||||
The easiest way to play with this module is to use :class:`parsing.Parser`.
|
By using :py:meth:`parso.tree.NodeOrLeaf.get_code` on a module, you can get
|
||||||
:attr:`parsing.Parser.module` holds an instance of :class:`Module`:
|
back the 1-to-1 representation of the input given to the parser. This is
|
||||||
|
important if you want to refactor a parser tree.
|
||||||
|
|
||||||
>>> from parso import parse
|
>>> from parso import parse
|
||||||
>>> parser = parse('import os')
|
>>> parser = parse('import os')
|
||||||
@@ -23,6 +25,21 @@ Any subclasses of :class:`Scope`, including :class:`Module` has an attribute
|
|||||||
|
|
||||||
>>> list(module.iter_imports())
|
>>> list(module.iter_imports())
|
||||||
[<ImportName: import os@1,0>]
|
[<ImportName: import os@1,0>]
|
||||||
|
|
||||||
|
Changes to the Python Grammar
|
||||||
|
-----------------------------
|
||||||
|
|
||||||
|
A few things have changed when looking at Python grammar files:
|
||||||
|
|
||||||
|
- :class:`Param` does not exist in Python grammar files. It is essentially a
|
||||||
|
part of a ``parameters`` node. |parso| splits it up to make it easier to
|
||||||
|
analyse parameters. However this just makes it easier to deal with the syntax
|
||||||
|
tree, it doesn't actually change the valid syntax.
|
||||||
|
- A few nodes like `lambdef` and `lambdef_nocond` have been merged in the
|
||||||
|
syntax tree to make it easier to do deal with them.
|
||||||
|
|
||||||
|
Parser Tree Classes
|
||||||
|
-------------------
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import re
|
import re
|
||||||
@@ -79,6 +96,10 @@ class PythonMixin(object):
|
|||||||
__slots__ = ()
|
__slots__ = ()
|
||||||
|
|
||||||
def get_name_of_position(self, position):
|
def get_name_of_position(self, position):
|
||||||
|
"""
|
||||||
|
Given a (line, column) tuple, returns a :class`Name` or ``None`` if
|
||||||
|
there is no name at that position.
|
||||||
|
"""
|
||||||
for c in self.children:
|
for c in self.children:
|
||||||
if isinstance(c, Leaf):
|
if isinstance(c, Leaf):
|
||||||
if c.type == 'name' and c.start_pos <= position <= c.end_pos:
|
if c.type == 'name' and c.start_pos <= position <= c.end_pos:
|
||||||
@@ -97,6 +118,9 @@ class PythonLeaf(PythonMixin, Leaf):
|
|||||||
return split_prefix(self, self.get_start_pos_of_prefix())
|
return split_prefix(self, self.get_start_pos_of_prefix())
|
||||||
|
|
||||||
def get_start_pos_of_prefix(self):
|
def get_start_pos_of_prefix(self):
|
||||||
|
"""
|
||||||
|
Basically calls :py:meth:`parso.tree.NodeOrLeaf.get_start_pos_of_prefix`.
|
||||||
|
"""
|
||||||
# TODO it is really ugly that we have to override it. Maybe change
|
# TODO it is really ugly that we have to override it. Maybe change
|
||||||
# indent error leafs somehow? No idea how, though.
|
# indent error leafs somehow? No idea how, though.
|
||||||
previous_leaf = self.get_previous_leaf()
|
previous_leaf = self.get_previous_leaf()
|
||||||
@@ -343,7 +367,8 @@ class Module(Scope):
|
|||||||
|
|
||||||
def _iter_future_import_names(self):
|
def _iter_future_import_names(self):
|
||||||
"""
|
"""
|
||||||
:return list of str: A list of future import names.
|
:return: A list of future import names.
|
||||||
|
:rtype: list of str
|
||||||
"""
|
"""
|
||||||
# In Python it's not allowed to use future imports after the first
|
# In Python it's not allowed to use future imports after the first
|
||||||
# actual (non-future) statement. However this is not a linter here,
|
# actual (non-future) statement. However this is not a linter here,
|
||||||
@@ -370,8 +395,8 @@ class Module(Scope):
|
|||||||
|
|
||||||
def get_used_names(self):
|
def get_used_names(self):
|
||||||
"""
|
"""
|
||||||
Returns all the `Name` leafs that exist in this module. Tihs includes
|
Returns all the :class:`Name` leafs that exist in this module. This
|
||||||
both definitions and references of names.
|
includes both definitions and references of names.
|
||||||
"""
|
"""
|
||||||
if self._used_names is None:
|
if self._used_names is None:
|
||||||
# Don't directly use self._used_names to eliminate a lookup.
|
# Don't directly use self._used_names to eliminate a lookup.
|
||||||
@@ -410,7 +435,7 @@ class ClassOrFunc(Scope):
|
|||||||
|
|
||||||
def get_decorators(self):
|
def get_decorators(self):
|
||||||
"""
|
"""
|
||||||
:return list of Decorator:
|
:rtype: list of :class:`Decorator`
|
||||||
"""
|
"""
|
||||||
decorated = self.parent
|
decorated = self.parent
|
||||||
if decorated.type == 'decorated':
|
if decorated.type == 'decorated':
|
||||||
@@ -425,13 +450,6 @@ class ClassOrFunc(Scope):
|
|||||||
class Class(ClassOrFunc):
|
class Class(ClassOrFunc):
|
||||||
"""
|
"""
|
||||||
Used to store the parsed contents of a python class.
|
Used to store the parsed contents of a python class.
|
||||||
|
|
||||||
:param name: The Class name.
|
|
||||||
:type name: str
|
|
||||||
:param supers: The super classes of a Class.
|
|
||||||
:type supers: list
|
|
||||||
:param start_pos: The start position (line, column) of the class.
|
|
||||||
:type start_pos: tuple(int, int)
|
|
||||||
"""
|
"""
|
||||||
type = 'classdef'
|
type = 'classdef'
|
||||||
__slots__ = ()
|
__slots__ = ()
|
||||||
|
|||||||
@@ -4,12 +4,12 @@ from parso._compatibility import utf8_repr, encoding, py_version
|
|||||||
|
|
||||||
def search_ancestor(node, *node_types):
|
def search_ancestor(node, *node_types):
|
||||||
"""
|
"""
|
||||||
Recursively looks at the parents of a node and checks if the type names
|
Recursively looks at the parents of a node and returns the first found node
|
||||||
match.
|
that matches node_types. Returns ``None`` if no matching node is found.
|
||||||
|
|
||||||
:param node: The node that is looked at.
|
:param node: The ancestors of this node will be checked.
|
||||||
:param node_types: A tuple or a string of type names that are
|
:param node_types: type names that are searched for.
|
||||||
searched for.
|
:type node_types: tuple of str
|
||||||
"""
|
"""
|
||||||
while True:
|
while True:
|
||||||
node = node.parent
|
node = node.parent
|
||||||
@@ -22,6 +22,10 @@ class NodeOrLeaf(object):
|
|||||||
The base class for nodes and leaves.
|
The base class for nodes and leaves.
|
||||||
"""
|
"""
|
||||||
__slots__ = ()
|
__slots__ = ()
|
||||||
|
type = None
|
||||||
|
'''
|
||||||
|
The type is a string that typically matches the types of the grammar file.
|
||||||
|
'''
|
||||||
|
|
||||||
def get_root_node(self):
|
def get_root_node(self):
|
||||||
"""
|
"""
|
||||||
@@ -35,8 +39,8 @@ class NodeOrLeaf(object):
|
|||||||
|
|
||||||
def get_next_sibling(self):
|
def get_next_sibling(self):
|
||||||
"""
|
"""
|
||||||
The node immediately following the invocant in their parent's children
|
Returns the node immediately following this node in this parent's
|
||||||
list. If the invocant does not have a next sibling, it is None
|
children list. If this node does not have a next sibling, it is None
|
||||||
"""
|
"""
|
||||||
# Can't use index(); we need to test by identity
|
# Can't use index(); we need to test by identity
|
||||||
for i, child in enumerate(self.parent.children):
|
for i, child in enumerate(self.parent.children):
|
||||||
@@ -48,8 +52,9 @@ class NodeOrLeaf(object):
|
|||||||
|
|
||||||
def get_previous_sibling(self):
|
def get_previous_sibling(self):
|
||||||
"""
|
"""
|
||||||
The node/leaf immediately preceding the invocant in their parent's
|
Returns the node immediately preceding this node in this parent's
|
||||||
children list. If the invocant does not have a previous sibling, it is
|
children list. If this node does not have a previous sibling, it is
|
||||||
|
None.
|
||||||
None.
|
None.
|
||||||
"""
|
"""
|
||||||
# Can't use index(); we need to test by identity
|
# Can't use index(); we need to test by identity
|
||||||
@@ -62,7 +67,7 @@ class NodeOrLeaf(object):
|
|||||||
def get_previous_leaf(self):
|
def get_previous_leaf(self):
|
||||||
"""
|
"""
|
||||||
Returns the previous leaf in the parser tree.
|
Returns the previous leaf in the parser tree.
|
||||||
Raises an IndexError if it's the first element in the parser tree.
|
Returns `None` if this is the first element in the parser tree.
|
||||||
"""
|
"""
|
||||||
node = self
|
node = self
|
||||||
while True:
|
while True:
|
||||||
@@ -85,7 +90,7 @@ class NodeOrLeaf(object):
|
|||||||
def get_next_leaf(self):
|
def get_next_leaf(self):
|
||||||
"""
|
"""
|
||||||
Returns the next leaf in the parser tree.
|
Returns the next leaf in the parser tree.
|
||||||
Returns `None` if it's the last element in the parser tree.
|
Returns None if this is the last element in the parser tree.
|
||||||
"""
|
"""
|
||||||
node = self
|
node = self
|
||||||
while True:
|
while True:
|
||||||
@@ -135,19 +140,19 @@ class NodeOrLeaf(object):
|
|||||||
@abstractmethod
|
@abstractmethod
|
||||||
def get_first_leaf(self):
|
def get_first_leaf(self):
|
||||||
"""
|
"""
|
||||||
Returns the first leaf of a node or itself it's a leaf.
|
Returns the first leaf of a node or itself if this is a leaf.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
@abstractmethod
|
@abstractmethod
|
||||||
def get_last_leaf(self):
|
def get_last_leaf(self):
|
||||||
"""
|
"""
|
||||||
Returns the last leaf of a node or itself it's a leaf.
|
Returns the last leaf of a node or itself if this is a leaf.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
@abstractmethod
|
@abstractmethod
|
||||||
def get_code(self, include_prefix=True):
|
def get_code(self, include_prefix=True):
|
||||||
"""
|
"""
|
||||||
Returns the code that was the input of the parser.
|
Returns the code that was input the input for the parser for this node.
|
||||||
|
|
||||||
:param include_prefix: Removes the prefix (whitespace and comments) of
|
:param include_prefix: Removes the prefix (whitespace and comments) of
|
||||||
e.g. a statement.
|
e.g. a statement.
|
||||||
@@ -155,13 +160,27 @@ class NodeOrLeaf(object):
|
|||||||
|
|
||||||
|
|
||||||
class Leaf(NodeOrLeaf):
|
class Leaf(NodeOrLeaf):
|
||||||
|
'''
|
||||||
|
Leafs are basically tokens with a better API. Leafs exactly know where they
|
||||||
|
were defined and what text preceeds them.
|
||||||
|
'''
|
||||||
__slots__ = ('value', 'parent', 'line', 'column', 'prefix')
|
__slots__ = ('value', 'parent', 'line', 'column', 'prefix')
|
||||||
|
|
||||||
def __init__(self, value, start_pos, prefix=''):
|
def __init__(self, value, start_pos, prefix=''):
|
||||||
self.value = value
|
self.value = value
|
||||||
|
'''
|
||||||
|
:py:func:`str` The value of the current token.
|
||||||
|
'''
|
||||||
self.start_pos = start_pos
|
self.start_pos = start_pos
|
||||||
self.prefix = prefix
|
self.prefix = prefix
|
||||||
|
'''
|
||||||
|
:py:func:`str` Typically a mixture of whitespace and comments. Stuff
|
||||||
|
that is syntactically irrelevant for the syntax tree.
|
||||||
|
'''
|
||||||
self.parent = None
|
self.parent = None
|
||||||
|
'''
|
||||||
|
The parent :class:`BaseNode` of this leaf.
|
||||||
|
'''
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def start_pos(self):
|
def start_pos(self):
|
||||||
@@ -219,9 +238,7 @@ class TypedLeaf(Leaf):
|
|||||||
class BaseNode(NodeOrLeaf):
|
class BaseNode(NodeOrLeaf):
|
||||||
"""
|
"""
|
||||||
The super class for all nodes.
|
The super class for all nodes.
|
||||||
|
A node has children, a type and possibly a parent node.
|
||||||
If you create custom nodes, you will probably want to inherit from this
|
|
||||||
``BaseNode``.
|
|
||||||
"""
|
"""
|
||||||
__slots__ = ('children', 'parent')
|
__slots__ = ('children', 'parent')
|
||||||
type = None
|
type = None
|
||||||
@@ -230,7 +247,14 @@ class BaseNode(NodeOrLeaf):
|
|||||||
for c in children:
|
for c in children:
|
||||||
c.parent = self
|
c.parent = self
|
||||||
self.children = children
|
self.children = children
|
||||||
|
"""
|
||||||
|
A list of :class:`NodeOrLeaf` child nodes.
|
||||||
|
"""
|
||||||
self.parent = None
|
self.parent = None
|
||||||
|
'''
|
||||||
|
The parent :class:`BaseNode` of this leaf.
|
||||||
|
None if this is the root node.
|
||||||
|
'''
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def start_pos(self):
|
def start_pos(self):
|
||||||
|
|||||||
@@ -11,10 +11,10 @@ Version = namedtuple('Version', 'major, minor, micro')
|
|||||||
|
|
||||||
def split_lines(string, keepends=False):
|
def split_lines(string, keepends=False):
|
||||||
r"""
|
r"""
|
||||||
A str.splitlines for Python code. In contrast to Python's ``str.splitlines``,
|
Intended for Python code. In contrast to Python's :py:meth:`str.splitlines`,
|
||||||
looks at form feeds and other special characters as normal text. Just
|
looks at form feeds and other special characters as normal text. Just
|
||||||
splits ``\n`` and ``\r\n``.
|
splits ``\n`` and ``\r\n``.
|
||||||
Also different: Returns ``['']`` for an empty string input.
|
Also different: Returns ``[""]`` for an empty string input.
|
||||||
|
|
||||||
In Python 2.7 form feeds are used as normal characters when using
|
In Python 2.7 form feeds are used as normal characters when using
|
||||||
str.splitlines. However in Python 3 somewhere there was a decision to split
|
str.splitlines. However in Python 3 somewhere there was a decision to split
|
||||||
@@ -48,9 +48,14 @@ def split_lines(string, keepends=False):
|
|||||||
return re.split('\n|\r\n', string)
|
return re.split('\n|\r\n', string)
|
||||||
|
|
||||||
|
|
||||||
def python_bytes_to_unicode(source, default_encoding='utf-8', errors='strict'):
|
def python_bytes_to_unicode(source, encoding='utf-8', errors='strict'):
|
||||||
"""
|
"""
|
||||||
`errors` can be 'strict', 'replace' or 'ignore'.
|
Checks for unicode BOMs and PEP 263 encoding declarations. Then returns a
|
||||||
|
unicode object like in :py:meth:`bytes.decode`.
|
||||||
|
|
||||||
|
:param encoding: See :py:meth:`bytes.decode` documentation.
|
||||||
|
:param errors: See :py:meth:`bytes.decode` documentation. ``errors`` can be
|
||||||
|
``'strict'``, ``'replace'`` or ``'ignore'``.
|
||||||
"""
|
"""
|
||||||
def detect_encoding():
|
def detect_encoding():
|
||||||
"""
|
"""
|
||||||
@@ -70,7 +75,7 @@ def python_bytes_to_unicode(source, default_encoding='utf-8', errors='strict'):
|
|||||||
return possible_encoding.group(1)
|
return possible_encoding.group(1)
|
||||||
else:
|
else:
|
||||||
# the default if nothing else has been set -> PEP 263
|
# the default if nothing else has been set -> PEP 263
|
||||||
return default_encoding
|
return encoding
|
||||||
|
|
||||||
if isinstance(source, unicode):
|
if isinstance(source, unicode):
|
||||||
# only cast str/bytes
|
# only cast str/bytes
|
||||||
|
|||||||
2
setup.py
2
setup.py
@@ -14,7 +14,7 @@ readme = open('README.rst').read() + '\n\n' + open('CHANGELOG.rst').read()
|
|||||||
|
|
||||||
setup(name='parso',
|
setup(name='parso',
|
||||||
version=parso.__version__,
|
version=parso.__version__,
|
||||||
description='A Python parser written in Python.',
|
description='A Python Parser',
|
||||||
author=__AUTHOR__,
|
author=__AUTHOR__,
|
||||||
author_email=__AUTHOR_EMAIL__,
|
author_email=__AUTHOR_EMAIL__,
|
||||||
include_package_data=True,
|
include_package_data=True,
|
||||||
|
|||||||
Reference in New Issue
Block a user