diff --git a/jedi/api/__init__.py b/jedi/api/__init__.py index 3b04b271..159b00fb 100644 --- a/jedi/api/__init__.py +++ b/jedi/api/__init__.py @@ -14,7 +14,7 @@ import warnings import sys from jedi.parser.python import load_grammar -from jedi.parser import tree +from jedi.parser.python import tree from jedi.parser.diff import FastParser from jedi.parser.utils import save_parser from jedi import debug diff --git a/jedi/api/completion.py b/jedi/api/completion.py index f18815f9..7b3340dc 100644 --- a/jedi/api/completion.py +++ b/jedi/api/completion.py @@ -1,5 +1,5 @@ from jedi.parser import token -from jedi.parser import tree +from jedi.parser.python import tree from jedi import debug from jedi import settings from jedi.api import classes diff --git a/jedi/api/helpers.py b/jedi/api/helpers.py index 294a5871..97b08f57 100644 --- a/jedi/api/helpers.py +++ b/jedi/api/helpers.py @@ -7,7 +7,7 @@ from collections import namedtuple from jedi._compatibility import u from jedi.evaluate.helpers import evaluate_call_of_leaf from jedi import parser -from jedi.parser import tree +from jedi.parser.python import tree from jedi.parser import tokenize from jedi.cache import time_cache from jedi import common diff --git a/jedi/api/keywords.py b/jedi/api/keywords.py index 63627327..d7491030 100644 --- a/jedi/api/keywords.py +++ b/jedi/api/keywords.py @@ -4,7 +4,7 @@ import keyword from jedi._compatibility import is_py3, is_py35 from jedi import common from jedi.evaluate.filters import AbstractNameDefinition -from jedi.parser.tree import Leaf +from jedi.parser.python.tree import Leaf try: from pydoc_data import topics as pydoc_topics except ImportError: diff --git a/jedi/api/usages.py b/jedi/api/usages.py index f9dba6ec..9bca83d5 100644 --- a/jedi/api/usages.py +++ b/jedi/api/usages.py @@ -1,5 +1,5 @@ from jedi.api import classes -from jedi.parser import tree +from jedi.parser.python import tree from jedi.evaluate import imports from jedi.evaluate.filters import TreeNameDefinition from jedi.evaluate.representation import ModuleContext diff --git a/jedi/evaluate/__init__.py b/jedi/evaluate/__init__.py index c3b8992d..8f8f2856 100644 --- a/jedi/evaluate/__init__.py +++ b/jedi/evaluate/__init__.py @@ -63,7 +63,7 @@ that are not used are just being ignored. import copy import sys -from jedi.parser import tree +from jedi.parser.python import tree from jedi import debug from jedi.common import unite from jedi.evaluate import representation as er diff --git a/jedi/evaluate/analysis.py b/jedi/evaluate/analysis.py index d48a043c..2beb3f02 100644 --- a/jedi/evaluate/analysis.py +++ b/jedi/evaluate/analysis.py @@ -2,7 +2,7 @@ Module for statical analysis. """ from jedi import debug -from jedi.parser import tree +from jedi.parser.python import tree from jedi.evaluate.compiled import CompiledObject from jedi.common import unite diff --git a/jedi/evaluate/compiled/__init__.py b/jedi/evaluate/compiled/__init__.py index b524fe10..1f68db6b 100644 --- a/jedi/evaluate/compiled/__init__.py +++ b/jedi/evaluate/compiled/__init__.py @@ -10,7 +10,7 @@ from functools import partial from jedi._compatibility import builtins as _builtins, unicode from jedi import debug from jedi.cache import underscore_memoization, memoize_method -from jedi.parser.tree import Param, Operator +from jedi.parser.python.tree import Param, Operator from jedi.evaluate.helpers import FakeName from jedi.evaluate.filters import AbstractFilter, AbstractNameDefinition, \ ContextNameMixin diff --git a/jedi/evaluate/compiled/fake.py b/jedi/evaluate/compiled/fake.py index d4ebfa17..8efecc5d 100644 --- a/jedi/evaluate/compiled/fake.py +++ b/jedi/evaluate/compiled/fake.py @@ -10,7 +10,7 @@ import types from jedi._compatibility import is_py3, builtins, unicode, is_py34 from jedi.parser.python import parse -from jedi.parser import tree as pt +from jedi.parser.python import tree as pt modules = {} @@ -113,7 +113,7 @@ def _faked(module, obj, name): if faked_mod is None: return None, None - # Having the module as a `parser.tree.Module`, we need to scan + # Having the module as a `parser.python.tree.Module`, we need to scan # for methods. if name is None: if inspect.isbuiltin(obj) or inspect.isclass(obj): diff --git a/jedi/evaluate/compiled/mixed.py b/jedi/evaluate/compiled/mixed.py index 09a4d9af..55ef1efd 100644 --- a/jedi/evaluate/compiled/mixed.py +++ b/jedi/evaluate/compiled/mixed.py @@ -17,7 +17,7 @@ class MixedObject(object): """ A ``MixedObject`` is used in two ways: - 1. It uses the default logic of ``parser.tree`` objects, + 1. It uses the default logic of ``parser.python.tree`` objects, 2. except for getattr calls. The names dicts are generated in a fashion like ``CompiledObject``. diff --git a/jedi/evaluate/docstrings.py b/jedi/evaluate/docstrings.py index f2eb0457..106a7879 100644 --- a/jedi/evaluate/docstrings.py +++ b/jedi/evaluate/docstrings.py @@ -23,7 +23,7 @@ from jedi.common import unite from jedi.evaluate import context from jedi.evaluate.cache import memoize_default from jedi.parser.python import parse -from jedi.parser.tree import search_ancestor +from jedi.parser.python.tree import search_ancestor from jedi.common import indent_block from jedi.evaluate.iterable import SequenceLiteralContext, FakeSequence diff --git a/jedi/evaluate/dynamic.py b/jedi/evaluate/dynamic.py index 8ec1b128..584ab9ed 100644 --- a/jedi/evaluate/dynamic.py +++ b/jedi/evaluate/dynamic.py @@ -17,7 +17,7 @@ It works as follows: - execute these calls and check the input. This work with a ``ParamListener``. """ -from jedi.parser import tree +from jedi.parser.python import tree from jedi import settings from jedi import debug from jedi.evaluate.cache import memoize_default diff --git a/jedi/evaluate/filters.py b/jedi/evaluate/filters.py index 33b8c567..2a7dc50e 100644 --- a/jedi/evaluate/filters.py +++ b/jedi/evaluate/filters.py @@ -4,7 +4,7 @@ are needed for name resolution. """ from abc import abstractmethod -from jedi.parser.tree import search_ancestor +from jedi.parser.python.tree import search_ancestor from jedi.evaluate import flow_analysis from jedi.common import to_list, unite diff --git a/jedi/evaluate/finder.py b/jedi/evaluate/finder.py index fe7cb9f8..8d624da0 100644 --- a/jedi/evaluate/finder.py +++ b/jedi/evaluate/finder.py @@ -15,7 +15,7 @@ Unfortunately every other thing is being ignored (e.g. a == '' would be easy to check for -> a is a string). There's big potential in these checks. """ -from jedi.parser import tree +from jedi.parser.python import tree from jedi import debug from jedi.common import unite from jedi import settings diff --git a/jedi/evaluate/helpers.py b/jedi/evaluate/helpers.py index 1eb6a6c2..36c1868c 100644 --- a/jedi/evaluate/helpers.py +++ b/jedi/evaluate/helpers.py @@ -2,7 +2,7 @@ import copy from itertools import chain from contextlib import contextmanager -from jedi.parser import tree +from jedi.parser.python import tree def deep_ast_copy(obj): diff --git a/jedi/evaluate/imports.py b/jedi/evaluate/imports.py index 6fee92b8..a0876786 100644 --- a/jedi/evaluate/imports.py +++ b/jedi/evaluate/imports.py @@ -21,7 +21,7 @@ from jedi import debug from jedi import settings from jedi.common import source_to_unicode, unite from jedi.parser.diff import FastParser -from jedi.parser import tree +from jedi.parser.python import tree from jedi.parser.utils import save_parser, load_parser, parser_cache from jedi.evaluate import sys_path from jedi.evaluate import helpers diff --git a/jedi/evaluate/param.py b/jedi/evaluate/param.py index b9161c6c..7563494f 100644 --- a/jedi/evaluate/param.py +++ b/jedi/evaluate/param.py @@ -3,7 +3,7 @@ from collections import defaultdict from jedi._compatibility import zip_longest from jedi import debug from jedi import common -from jedi.parser import tree +from jedi.parser.python import tree from jedi.evaluate import iterable from jedi.evaluate import analysis from jedi.evaluate import context diff --git a/jedi/evaluate/pep0484.py b/jedi/evaluate/pep0484.py index 5f543c9a..098af808 100644 --- a/jedi/evaluate/pep0484.py +++ b/jedi/evaluate/pep0484.py @@ -22,8 +22,8 @@ x support for type hint comments for functions, `# type: (int, str) -> int`. import itertools import os -from jedi.parser import ParserSyntaxError, tree -from jedi.parser.python import parse +from jedi.parser import ParserSyntaxError +from jedi.parser.python import parse, tree from jedi.common import unite from jedi.evaluate.cache import memoize_default from jedi.evaluate import compiled diff --git a/jedi/evaluate/precedence.py b/jedi/evaluate/precedence.py index 76501e1e..51ceb153 100644 --- a/jedi/evaluate/precedence.py +++ b/jedi/evaluate/precedence.py @@ -4,7 +4,7 @@ Handles operator precedence. import operator as op from jedi._compatibility import unicode -from jedi.parser import tree +from jedi.parser.python import tree from jedi import debug from jedi.evaluate.compiled import CompiledObject, create, builtin_from_name from jedi.evaluate import analysis diff --git a/jedi/evaluate/representation.py b/jedi/evaluate/representation.py index 941c21f5..fdb8ef0f 100644 --- a/jedi/evaluate/representation.py +++ b/jedi/evaluate/representation.py @@ -1,5 +1,5 @@ """ -Like described in the :mod:`jedi.parser.tree` module, +Like described in the :mod:`jedi.parser.python.tree` module, there's a need for an ast like module to represent the states of parsed modules. @@ -42,7 +42,7 @@ import re from itertools import chain from jedi._compatibility import use_metaclass -from jedi.parser import tree +from jedi.parser.python import tree from jedi import debug from jedi import common from jedi.evaluate.cache import memoize_default, CachedMetaClass, NO_DEFAULT diff --git a/jedi/evaluate/sys_path.py b/jedi/evaluate/sys_path.py index ccecaccb..b5f31e9b 100644 --- a/jedi/evaluate/sys_path.py +++ b/jedi/evaluate/sys_path.py @@ -4,7 +4,7 @@ import sys from jedi.evaluate.site import addsitedir from jedi._compatibility import exec_function, unicode -from jedi.parser import tree +from jedi.parser.python import tree from jedi.parser import ParserWithRecovery from jedi.evaluate.cache import memoize_default from jedi import debug diff --git a/jedi/parser/diff.py b/jedi/parser/diff.py index 5842fcfb..e7da51b2 100644 --- a/jedi/parser/diff.py +++ b/jedi/parser/diff.py @@ -13,7 +13,7 @@ from jedi._compatibility import use_metaclass from jedi import settings from jedi.common import splitlines from jedi.parser import ParserWithRecovery -from jedi.parser.tree import EndMarker +from jedi.parser.python.tree import EndMarker from jedi.parser.utils import parser_cache from jedi import debug from jedi.parser.tokenize import (generate_tokens, NEWLINE, TokenInfo, diff --git a/jedi/parser/python/parser.py b/jedi/parser/python/parser.py index aa34dccb..4e2e71fd 100644 --- a/jedi/parser/python/parser.py +++ b/jedi/parser/python/parser.py @@ -1,6 +1,6 @@ import re -from jedi.parser import tree +from jedi.parser.python import tree from jedi.parser import tokenize from jedi.parser.token import (DEDENT, INDENT, ENDMARKER, NEWLINE, NUMBER, STRING, tok_name) diff --git a/jedi/parser/python/tree.py b/jedi/parser/python/tree.py new file mode 100644 index 00000000..38b63d0c --- /dev/null +++ b/jedi/parser/python/tree.py @@ -0,0 +1,1631 @@ +""" +If you know what an syntax tree is, you'll see that this module is pretty much +that. The classes represent syntax elements like functions and imports. + +This is the "business logic" part of the parser. There's a lot of logic here +that makes it easier for Jedi (and other libraries) to deal with a Python syntax +tree. + +By using `get_code` on a module, you can get back the 1-to-1 representation of +the input given to the parser. This is important if you are using refactoring. + +The easiest way to play with this module is to use :class:`parsing.Parser`. +:attr:`parsing.Parser.module` holds an instance of :class:`Module`: + +>>> from jedi._compatibility import u +>>> from jedi.parser.python import load_grammar +>>> from jedi.parser import ParserWithRecovery +>>> parser = ParserWithRecovery(load_grammar(), u('import os'), 'example.py') +>>> submodule = parser.module +>>> submodule + + +Any subclasses of :class:`Scope`, including :class:`Module` has an attribute +:attr:`imports `: + +>>> submodule.imports +[] + +See also :attr:`Scope.subscopes` and :attr:`Scope.statements`. + +For static analysis purposes there exists a method called +``nodes_to_execute`` on all nodes and leaves. It's documented in the static +anaylsis documentation. +""" +import os +import re +from inspect import cleandoc +from itertools import chain +import textwrap +import abc + +from jedi._compatibility import (Python3Method, encoding, is_py3, utf8_repr, + literal_eval, unicode) + + +def _safe_literal_eval(value): + first_two = value[:2].lower() + if first_two[0] == 'f' or first_two in ('fr', 'rf'): + # literal_eval is not able to resovle f literals. We have to do that + # manually in a later stage + return '' + + try: + return literal_eval(value) + except SyntaxError: + # It's possible to create syntax errors with literals like rb'' in + # Python 2. This should not be possible and in that case just return an + # empty string. + # Before Python 3.3 there was a more strict definition in which order + # you could define literals. + return '' + + +def search_ancestor(node, node_type_or_types): + if not isinstance(node_type_or_types, (list, tuple)): + node_type_or_types = (node_type_or_types,) + + while True: + node = node.parent + if node is None or node.type in node_type_or_types: + return node + + +class DocstringMixin(object): + __slots__ = () + + @property + def raw_doc(self): + """ Returns a cleaned version of the docstring token. """ + if isinstance(self, Module): + node = self.children[0] + elif isinstance(self, ClassOrFunc): + node = self.children[self.children.index(':') + 1] + if node.type == 'suite': # Normally a suite + node = node.children[1] # -> NEWLINE stmt + else: # ExprStmt + simple_stmt = self.parent + c = simple_stmt.parent.children + index = c.index(simple_stmt) + if not index: + return '' + node = c[index - 1] + + if node.type == 'simple_stmt': + node = node.children[0] + + if node.type == 'string': + # TODO We have to check next leaves until there are no new + # leaves anymore that might be part of the docstring. A + # docstring can also look like this: ``'foo' 'bar' + # Returns a literal cleaned version of the ``Token``. + cleaned = cleandoc(_safe_literal_eval(node.value)) + # Since we want the docstr output to be always unicode, just + # force it. + if is_py3 or isinstance(cleaned, unicode): + return cleaned + else: + return unicode(cleaned, 'UTF-8', 'replace') + return '' + + +class Base(object): + """ + This is just here to have an isinstance check, which is also used on + evaluate classes. But since they have sometimes a special type of + delegation, it is important for those classes to override this method. + + I know that there is a chance to do such things with __instancecheck__, but + since Python 2.5 doesn't support it, I decided to do it this way. + """ + __slots__ = () + + def get_root_node(self): + scope = self + while scope.parent is not None: + scope = scope.parent + return scope + + def get_parent_scope(self, include_flows=False): + """ + Returns the underlying scope. + """ + scope = self.parent + while scope is not None: + if include_flows and isinstance(scope, Flow): + return scope + if scope.is_scope(): + break + scope = scope.parent + return scope + + def get_definition(self): + if self.type in ('newline', 'endmarker'): + raise ValueError('Cannot get the indentation of whitespace or indentation.') + scope = self + while scope.parent is not None: + parent = scope.parent + if isinstance(scope, (Node, Leaf)) and parent.type != 'simple_stmt': + if scope.type == 'testlist_comp': + try: + if isinstance(scope.children[1], CompFor): + return scope.children[1] + except IndexError: + pass + scope = parent + else: + break + return scope + + def assignment_indexes(self): + """ + Returns an array of tuple(int, node) of the indexes that are used in + tuple assignments. + + For example if the name is ``y`` in the following code:: + + x, (y, z) = 2, '' + + would result in ``[(1, xyz_node), (0, yz_node)]``. + """ + indexes = [] + node = self.parent + compare = self + while node is not None: + if node.type in ('testlist_comp', 'testlist_star_expr', 'exprlist'): + for i, child in enumerate(node.children): + if child == compare: + indexes.insert(0, (int(i / 2), node)) + break + else: + raise LookupError("Couldn't find the assignment.") + elif isinstance(node, (ExprStmt, CompFor)): + break + + compare = node + node = node.parent + return indexes + + def is_scope(self): + # Default is not being a scope. Just inherit from Scope. + return False + + @abc.abstractmethod + def nodes_to_execute(self, last_added=False): + raise NotImplementedError() + + def get_next_sibling(self): + """ + The node immediately following the invocant in their parent's children + list. If the invocant does not have a next sibling, it is None + """ + # Can't use index(); we need to test by identity + for i, child in enumerate(self.parent.children): + if child is self: + try: + return self.parent.children[i + 1] + except IndexError: + return None + + def get_previous_sibling(self): + """ + The node/leaf immediately preceding the invocant in their parent's + children list. If the invocant does not have a previous sibling, it is + None. + """ + # Can't use index(); we need to test by identity + for i, child in enumerate(self.parent.children): + if child is self: + if i == 0: + return None + return self.parent.children[i - 1] + + def get_previous_leaf(self): + """ + Returns the previous leaf in the parser tree. + Raises an IndexError if it's the first element. + """ + node = self + while True: + c = node.parent.children + i = c.index(node) + if i == 0: + node = node.parent + if node.parent is None: + return None + else: + node = c[i - 1] + break + + while True: + try: + node = node.children[-1] + except AttributeError: # A Leaf doesn't have children. + return node + + def get_next_leaf(self): + """ + Returns the previous leaf in the parser tree. + Raises an IndexError if it's the last element. + """ + node = self + while True: + c = node.parent.children + i = c.index(node) + if i == len(c) - 1: + node = node.parent + if node.parent is None: + return None + else: + node = c[i + 1] + break + + while True: + try: + node = node.children[0] + except AttributeError: # A Leaf doesn't have children. + return node + + +class Leaf(Base): + __slots__ = ('value', 'parent', 'line', 'indent', 'prefix') + + def __init__(self, value, start_pos, prefix=''): + self.value = value + self.start_pos = start_pos + self.prefix = prefix + self.parent = None + + @property + def start_pos(self): + return self.line, self.indent + + @start_pos.setter + def start_pos(self, value): + self.line = value[0] + self.indent = value[1] + + def get_start_pos_of_prefix(self): + previous_leaf = self.get_previous_leaf() + if previous_leaf is None: + return self.line - self.prefix.count('\n'), 0 # It's the first leaf. + return previous_leaf.end_pos + + @property + def end_pos(self): + return self.line, self.indent + len(self.value) + + def move(self, line_offset): + self.line += line_offset + + def get_first_leaf(self): + return self + + def get_last_leaf(self): + return self + + def get_code(self, normalized=False, include_prefix=True): + if normalized: + return self.value + if include_prefix: + return self.prefix + self.value + else: + return self.value + + def nodes_to_execute(self, last_added=False): + return [] + + @utf8_repr + def __repr__(self): + return "<%s: %s start=%s>" % (type(self).__name__, self.value, self.start_pos) + + +class LeafWithNewLines(Leaf): + __slots__ = () + + @property + def end_pos(self): + """ + Literals and whitespace end_pos are more complicated than normal + end_pos, because the containing newlines may change the indexes. + """ + lines = self.value.split('\n') + end_pos_line = self.line + len(lines) - 1 + # Check for multiline token + if self.line == end_pos_line: + end_pos_indent = self.indent + len(lines[-1]) + else: + end_pos_indent = len(lines[-1]) + return end_pos_line, end_pos_indent + + @utf8_repr + def __repr__(self): + return "<%s: %r>" % (type(self).__name__, self.value) + + +class EndMarker(Leaf): + __slots__ = () + type = 'endmarker' + + +class Newline(LeafWithNewLines): + """Contains NEWLINE and ENDMARKER tokens.""" + __slots__ = () + type = 'newline' + + @utf8_repr + def __repr__(self): + return "<%s: %s>" % (type(self).__name__, repr(self.value)) + + +class Name(Leaf): + """ + A string. Sometimes it is important to know if the string belongs to a name + or not. + """ + type = 'name' + __slots__ = () + + def __str__(self): + return self.value + + def __unicode__(self): + return self.value + + def __repr__(self): + return "<%s: %s@%s,%s>" % (type(self).__name__, self.value, + self.line, self.indent) + + def is_definition(self): + if self.parent.type in ('power', 'atom_expr'): + # In `self.x = 3` self is not a definition, but x is. + return False + + stmt = self.get_definition() + if stmt.type in ('funcdef', 'classdef', 'file_input', 'param'): + return self == stmt.name + elif stmt.type == 'for_stmt': + return self.start_pos < stmt.children[2].start_pos + elif stmt.type == 'try_stmt': + return self.get_previous_sibling() == 'as' + else: + return stmt.type in ('expr_stmt', 'import_name', 'import_from', + 'comp_for', 'with_stmt') \ + and self in stmt.get_defined_names() + + def nodes_to_execute(self, last_added=False): + if last_added is False: + yield self + + +class Literal(LeafWithNewLines): + __slots__ = () + + def eval(self): + return _safe_literal_eval(self.value) + + +class Number(Literal): + type = 'number' + __slots__ = () + + +class String(Literal): + type = 'string' + __slots__ = () + + +class Operator(Leaf): + type = 'operator' + __slots__ = () + + def __str__(self): + return self.value + + def __eq__(self, other): + """ + Make comparisons with strings easy. + Improves the readability of the parser. + """ + if isinstance(other, Operator): + return self is other + else: + return self.value == other + + def __ne__(self, other): + """Python 2 compatibility.""" + return self.value != other + + def __hash__(self): + return hash(self.value) + + +class Keyword(Leaf): + type = 'keyword' + __slots__ = () + + def __eq__(self, other): + """ + Make comparisons with strings easy. + Improves the readability of the parser. + """ + if isinstance(other, Keyword): + return self is other + return self.value == other + + def __ne__(self, other): + """Python 2 compatibility.""" + return not self.__eq__(other) + + def __hash__(self): + return hash(self.value) + + +class BaseNode(Base): + """ + The super class for Scope, Import, Name and Statement. Every object in + the parser tree inherits from this class. + """ + __slots__ = ('children', 'parent') + type = None + + def __init__(self, children): + """ + Initialize :class:`BaseNode`. + + :param children: The module in which this Python object locates. + """ + for c in children: + c.parent = self + self.children = children + self.parent = None + + def move(self, line_offset): + """ + Move the Node's start_pos. + """ + for c in self.children: + c.move(line_offset) + + @property + def start_pos(self): + return self.children[0].start_pos + + def get_start_pos_of_prefix(self): + return self.children[0].get_start_pos_of_prefix() + + @property + def end_pos(self): + return self.children[-1].end_pos + + def _get_code_for_children(self, children, normalized, include_prefix): + # TODO implement normalized (depending on context). + if include_prefix: + return "".join(c.get_code(normalized) for c in children) + else: + first = children[0].get_code(include_prefix=False) + return first + "".join(c.get_code(normalized) for c in children[1:]) + + def get_code(self, normalized=False, include_prefix=True): + return self._get_code_for_children(self.children, normalized, include_prefix) + + @Python3Method + def name_for_position(self, position): + for c in self.children: + if isinstance(c, Leaf): + if isinstance(c, Name) and c.start_pos <= position <= c.end_pos: + return c + else: + result = c.name_for_position(position) + if result is not None: + return result + return None + + def get_leaf_for_position(self, position, include_prefixes=False): + def binary_search(lower, upper): + if lower == upper: + element = self.children[lower] + if not include_prefixes and position < element.start_pos: + # We're on a prefix. + return None + # In case we have prefixes, a leaf always matches + try: + return element.get_leaf_for_position(position, include_prefixes) + except AttributeError: + return element + + + index = int((lower + upper) / 2) + element = self.children[index] + if position <= element.end_pos: + return binary_search(lower, index) + else: + return binary_search(index + 1, upper) + + if not ((1, 0) <= position <= self.children[-1].end_pos): + raise ValueError('Please provide a position that exists within this node.') + return binary_search(0, len(self.children) - 1) + + @Python3Method + def get_statement_for_position(self, pos): + for c in self.children: + if c.start_pos <= pos <= c.end_pos: + if c.type not in ('decorated', 'simple_stmt', 'suite') \ + and not isinstance(c, (Flow, ClassOrFunc)): + return c + else: + try: + return c.get_statement_for_position(pos) + except AttributeError: + pass # Must be a non-scope + return None + + def get_first_leaf(self): + return self.children[0].get_first_leaf() + + def get_last_leaf(self): + return self.children[-1].get_last_leaf() + + def get_following_comment_same_line(self): + """ + returns (as string) any comment that appears on the same line, + after the node, including the # + """ + try: + if self.type == 'for_stmt': + whitespace = self.children[5].get_first_leaf().prefix + elif self.type == 'with_stmt': + whitespace = self.children[3].get_first_leaf().prefix + else: + whitespace = self.get_last_leaf().get_next_leaf().prefix + except AttributeError: + return None + except ValueError: + # TODO in some particular cases, the tree doesn't seem to be linked + # correctly + return None + if "#" not in whitespace: + return None + comment = whitespace[whitespace.index("#"):] + if "\r" in comment: + comment = comment[:comment.index("\r")] + if "\n" in comment: + comment = comment[:comment.index("\n")] + return comment + + @utf8_repr + def __repr__(self): + code = self.get_code().replace('\n', ' ').strip() + if not is_py3: + code = code.encode(encoding, 'replace') + return "<%s: %s@%s,%s>" % \ + (type(self).__name__, code, self.start_pos[0], self.start_pos[1]) + + +class Node(BaseNode): + """Concrete implementation for interior nodes.""" + __slots__ = ('type',) + + _IGNORE_EXECUTE_NODES = set([ + 'suite', 'subscriptlist', 'subscript', 'simple_stmt', 'sliceop', + 'testlist_comp', 'dictorsetmaker', 'trailer', 'decorators', + 'decorated', 'arglist', 'argument', 'exprlist', 'testlist', + 'testlist_safe', 'testlist1' + ]) + + def __init__(self, type, children): + """ + Initializer. + + Takes a type constant (a symbol number >= 256), a sequence of + child nodes, and an optional context keyword argument. + + As a side effect, the parent pointers of the children are updated. + """ + super(Node, self).__init__(children) + self.type = type + + def nodes_to_execute(self, last_added=False): + """ + For static analysis. + """ + result = [] + if self.type not in Node._IGNORE_EXECUTE_NODES and not last_added: + result.append(self) + last_added = True + + for child in self.children: + result += child.nodes_to_execute(last_added) + return result + + def __repr__(self): + return "%s(%s, %r)" % (self.__class__.__name__, self.type, self.children) + + +class ErrorNode(BaseNode): + """ + TODO doc + """ + __slots__ = () + type = 'error_node' + + def nodes_to_execute(self, last_added=False): + return [] + + +class ErrorLeaf(LeafWithNewLines): + """ + TODO doc + """ + __slots__ = ('original_type') + type = 'error_leaf' + + def __init__(self, original_type, value, start_pos, prefix=''): + super(ErrorLeaf, self).__init__(value, start_pos, prefix) + self.original_type = original_type + + def __repr__(self): + return "<%s: %s:%s, %s)>" % \ + (type(self).__name__, self.original_type, repr(self.value), self.start_pos) + + +class Scope(BaseNode, DocstringMixin): + """ + Super class for the parser tree, which represents the state of a python + text file. + A Scope manages and owns its subscopes, which are classes and functions, as + well as variables and imports. It is used to access the structure of python + files. + + :param start_pos: The position (line and column) of the scope. + :type start_pos: tuple(int, int) + """ + __slots__ = () + + def __init__(self, children): + super(Scope, self).__init__(children) + + @property + def returns(self): + # Needed here for fast_parser, because the fast_parser splits and + # returns will be in "normal" modules. + return self._search_in_scope(ReturnStmt) + + @property + def subscopes(self): + return self._search_in_scope(Scope) + + @property + def flows(self): + return self._search_in_scope(Flow) + + @property + def imports(self): + return self._search_in_scope(Import) + + @Python3Method + def _search_in_scope(self, typ): + def scan(children): + elements = [] + for element in children: + if isinstance(element, typ): + elements.append(element) + if element.type in ('suite', 'simple_stmt', 'decorated') \ + or isinstance(element, Flow): + elements += scan(element.children) + return elements + + return scan(self.children) + + @property + def statements(self): + return self._search_in_scope((ExprStmt, KeywordStatement)) + + def is_scope(self): + return True + + def __repr__(self): + try: + name = self.path + except AttributeError: + try: + name = self.name + except AttributeError: + name = self.command + + return "<%s: %s@%s-%s>" % (type(self).__name__, name, + self.start_pos[0], self.end_pos[0]) + + def walk(self): + yield self + for s in self.subscopes: + for scope in s.walk(): + yield scope + + for r in self.statements: + while isinstance(r, Flow): + for scope in r.walk(): + yield scope + r = r.next + + +class Module(Scope): + """ + The top scope, which is always a module. + Depending on the underlying parser this may be a full module or just a part + of a module. + """ + __slots__ = ('path', 'used_names', '_name') + type = 'file_input' + + def __init__(self, children): + """ + Initialize :class:`Module`. + + :type path: str + :arg path: File path to this module. + + .. todo:: Document `top_module`. + """ + super(Module, self).__init__(children) + self.path = None # Set later. + + @property + def name(self): + """ This is used for the goto functions. """ + if self.path is None: + string = '' # no path -> empty name + else: + sep = (re.escape(os.path.sep),) * 2 + r = re.search(r'([^%s]*?)(%s__init__)?(\.py|\.so)?$' % sep, self.path) + # Remove PEP 3149 names + string = re.sub('\.[a-z]+-\d{2}[mud]{0,3}$', '', r.group(1)) + # Positions are not real, but a module starts at (1, 0) + p = (1, 0) + name = Name(string, p) + name.parent = self + return name + + @property + def has_explicit_absolute_import(self): + """ + Checks if imports in this module are explicitly absolute, i.e. there + is a ``__future__`` import. + """ + # TODO this is a strange scan and not fully correct. I think Python's + # parser does it in a different way and scans for the first + # statement/import with a tokenizer (to check for syntax changes like + # the future print statement). + for imp in self.imports: + if imp.type == 'import_from' and imp.level == 0: + for path in imp.paths(): + if [str(name) for name in path] == ['__future__', 'absolute_import']: + return True + return False + + def nodes_to_execute(self, last_added=False): + # Yield itself, class needs to be executed for decorator checks. + result = [] + for child in self.children: + result += child.nodes_to_execute() + return result + + +class Decorator(BaseNode): + type = 'decorator' + __slots__ = () + + def nodes_to_execute(self, last_added=False): + if self.children[-2] == ')': + node = self.children[-3] + if node != '(': + return node.nodes_to_execute() + return [] + + +class ClassOrFunc(Scope): + __slots__ = () + + @property + def name(self): + return self.children[1] + + def get_decorators(self): + decorated = self.parent + if decorated.type == 'decorated': + if decorated.children[0].type == 'decorators': + return decorated.children[0].children + else: + return decorated.children[:1] + else: + return [] + + +class Class(ClassOrFunc): + """ + Used to store the parsed contents of a python class. + + :param name: The Class name. + :type name: str + :param supers: The super classes of a Class. + :type supers: list + :param start_pos: The start position (line, column) of the class. + :type start_pos: tuple(int, int) + """ + type = 'classdef' + __slots__ = () + + def __init__(self, children): + super(Class, self).__init__(children) + + def get_super_arglist(self): + if self.children[2] != '(': # Has no parentheses + return None + else: + if self.children[3] == ')': # Empty parentheses + return None + else: + return self.children[3] + + @property + def doc(self): + """ + Return a document string including call signature of __init__. + """ + docstr = self.raw_doc + for sub in self.subscopes: + if str(sub.name) == '__init__': + return '%s\n\n%s' % ( + sub.get_call_signature(func_name=self.name), docstr) + return docstr + + def nodes_to_execute(self, last_added=False): + # Yield itself, class needs to be executed for decorator checks. + yield self + # Super arguments. + arglist = self.get_super_arglist() + try: + children = arglist.children + except AttributeError: + if arglist is not None: + for node_to_execute in arglist.nodes_to_execute(): + yield node_to_execute + else: + for argument in children: + if argument.type == 'argument': + # metaclass= or list comprehension or */** + raise NotImplementedError('Metaclasses not implemented') + else: + for node_to_execute in argument.nodes_to_execute(): + yield node_to_execute + + # care for the class suite: + for node in self.children[self.children.index(':'):]: + # This could be easier without the fast parser. But we need to find + # the position of the colon, because everything after it can be a + # part of the class, not just its suite. + for node_to_execute in node.nodes_to_execute(): + yield node_to_execute + + +def _create_params(parent, argslist_list): + """ + `argslist_list` is a list that can contain an argslist as a first item, but + most not. It's basically the items between the parameter brackets (which is + at most one item). + This function modifies the parser structure. It generates `Param` objects + from the normal ast. Those param objects do not exist in a normal ast, but + make the evaluation of the ast tree so much easier. + You could also say that this function replaces the argslist node with a + list of Param objects. + """ + def check_python2_nested_param(node): + """ + Python 2 allows params to look like ``def x(a, (b, c))``, which is + basically a way of unpacking tuples in params. Python 3 has ditched + this behavior. Jedi currently just ignores those constructs. + """ + return node.type == 'tfpdef' and node.children[0] == '(' + + try: + first = argslist_list[0] + except IndexError: + return [] + + if first.type in ('name', 'tfpdef'): + if check_python2_nested_param(first): + return [first] + else: + return [Param([first], parent)] + elif first == '*': + return [first] + else: # argslist is a `typedargslist` or a `varargslist`. + children = first.children + new_children = [] + start = 0 + # Start with offset 1, because the end is higher. + for end, child in enumerate(children + [None], 1): + if child is None or child == ',': + param_children = children[start:end] + if param_children: # Could as well be comma and then end. + if check_python2_nested_param(param_children[0]): + new_children += param_children + elif param_children[0] == '*' and param_children[1] == ',': + new_children += param_children + else: + new_children.append(Param(param_children, parent)) + start = end + return new_children + + +class Function(ClassOrFunc): + """ + Used to store the parsed contents of a python function. + + Children: + 0) + 1) + 2) parameter list (including open-paren and close-paren s) + 3 or 5) + 4 or 6) Node() representing function body + 3) -> (if annotation is also present) + 4) annotation (if present) + """ + type = 'funcdef' + + def __init__(self, children): + super(Function, self).__init__(children) + parameters = self.children[2] # After `def foo` + parameters.children[1:-1] = _create_params(parameters, parameters.children[1:-1]) + + @property + def params(self): + return [p for p in self.children[2].children if p.type == 'param'] + + @property + def name(self): + return self.children[1] # First token after `def` + + @property + def yields(self): + # TODO This is incorrect, yields are also possible in a statement. + return self._search_in_scope(YieldExpr) + + def is_generator(self): + return bool(self.yields) + + def annotation(self): + try: + if self.children[3] == "->": + return self.children[4] + assert self.children[3] == ":" + return None + except IndexError: + return None + + def get_call_signature(self, width=72, func_name=None): + """ + Generate call signature of this function. + + :param width: Fold lines if a line is longer than this value. + :type width: int + :arg func_name: Override function name when given. + :type func_name: str + + :rtype: str + """ + func_name = func_name or self.name + code = unicode(func_name) + self._get_paramlist_code() + return '\n'.join(textwrap.wrap(code, width)) + + def _get_paramlist_code(self): + return self.children[2].get_code() + + @property + def doc(self): + """ Return a document string including call signature. """ + docstr = self.raw_doc + return '%s\n\n%s' % (self.get_call_signature(), docstr) + + def nodes_to_execute(self, last_added=False): + # Yield itself, functions needs to be executed for decorator checks. + yield self + for param in self.params: + if param.default is not None: + yield param.default + # care for the function suite: + for node in self.children[4:]: + # This could be easier without the fast parser. The fast parser + # allows that the 4th position is empty or that there's even a + # fifth element (another function/class). So just scan everything + # after colon. + for node_to_execute in node.nodes_to_execute(): + yield node_to_execute + + +class Lambda(Function): + """ + Lambdas are basically trimmed functions, so give it the same interface. + + Children: + 0) + *) for each argument x + -2) + -1) Node() representing body + """ + type = 'lambda' + __slots__ = () + + def __init__(self, children): + # We don't want to call the Function constructor, call its parent. + super(Function, self).__init__(children) + lst = self.children[1:-2] # Everything between `lambda` and the `:` operator is a parameter. + self.children[1:-2] = _create_params(self, lst) + + @property + def name(self): + # Borrow the position of the AST node. + return Name('', self.children[0].start_pos) + + def _get_paramlist_code(self): + return '(' + ''.join(param.get_code() for param in self.params).strip() + ')' + + @property + def params(self): + return self.children[1:-2] + + def is_generator(self): + return False + + def annotation(self): + # lambda functions do not support annotations + return None + + @property + def yields(self): + return [] + + def nodes_to_execute(self, last_added=False): + for param in self.params: + if param.default is not None: + yield param.default + # Care for the lambda test (last child): + for node_to_execute in self.children[-1].nodes_to_execute(): + yield node_to_execute + + def __repr__(self): + return "<%s@%s>" % (self.__class__.__name__, self.start_pos) + + +class Flow(BaseNode): + __slots__ = () + FLOW_KEYWORDS = ( + 'try', 'except', 'finally', 'else', 'if', 'elif', 'with', 'for', 'while' + ) + + def nodes_to_execute(self, last_added=False): + for child in self.children: + for node_to_execute in child.nodes_to_execute(): + yield node_to_execute + + def get_branch_keyword(self, node): + start_pos = node.start_pos + if not (self.start_pos < start_pos <= self.end_pos): + raise ValueError('The node is not part of the flow.') + + keyword = None + for i, child in enumerate(self.children): + if start_pos < child.start_pos: + return keyword + first_leaf = child.get_first_leaf() + if first_leaf in self.FLOW_KEYWORDS: + keyword = first_leaf + return 0 + + +class IfStmt(Flow): + type = 'if_stmt' + __slots__ = () + + def check_nodes(self): + """ + Returns all the `test` nodes that are defined as x, here: + + if x: + pass + elif x: + pass + """ + for i, c in enumerate(self.children): + if c in ('elif', 'if'): + yield self.children[i + 1] + + def node_in_which_check_node(self, node): + """ + Returns the check node (see function above) that a node is contained + in. However if it the node is in the check node itself and not in the + suite return None. + """ + start_pos = node.start_pos + for check_node in reversed(list(self.check_nodes())): + if check_node.start_pos < start_pos: + if start_pos < check_node.end_pos: + return None + # In this case the node is within the check_node itself, + # not in the suite + else: + return check_node + + def node_after_else(self, node): + """ + Checks if a node is defined after `else`. + """ + for c in self.children: + if c == 'else': + if node.start_pos > c.start_pos: + return True + else: + return False + + +class WhileStmt(Flow): + type = 'while_stmt' + __slots__ = () + + +class ForStmt(Flow): + type = 'for_stmt' + __slots__ = () + + def get_input_node(self): + """ + Returns the input node ``y`` from: ``for x in y:``. + """ + return self.children[3] + + def defines_one_name(self): + """ + Returns True if only one name is returned: ``for x in y``. + Returns False if the for loop is more complicated: ``for x, z in y``. + + :returns: bool + """ + return self.children[1].type == 'name' + + +class TryStmt(Flow): + type = 'try_stmt' + __slots__ = () + + def except_clauses(self): + """ + Returns the ``test`` nodes found in ``except_clause`` nodes. + Returns ``[None]`` for except clauses without an exception given. + """ + for node in self.children: + if node.type == 'except_clause': + yield node.children[1] + elif node == 'except': + yield None + + def nodes_to_execute(self, last_added=False): + result = [] + for child in self.children[2::3]: + result += child.nodes_to_execute() + for child in self.children[0::3]: + if child.type == 'except_clause': + # Add the test node and ignore the `as NAME` definition. + result += child.children[1].nodes_to_execute() + return result + + +class WithStmt(Flow): + type = 'with_stmt' + __slots__ = () + + def get_defined_names(self): + names = [] + for with_item in self.children[1:-2:2]: + # Check with items for 'as' names. + if with_item.type == 'with_item': + names += _defined_names(with_item.children[2]) + return names + + def node_from_name(self, name): + node = name + while True: + node = node.parent + if node.type == 'with_item': + return node.children[0] + + def nodes_to_execute(self, last_added=False): + result = [] + for child in self.children[1::2]: + if child.type == 'with_item': + # Just ignore the `as EXPR` part - at least for now, because + # most times it's just a name. + child = child.children[0] + result += child.nodes_to_execute() + return result + + +class Import(BaseNode): + __slots__ = () + + def path_for_name(self, name): + try: + # The name may be an alias. If it is, just map it back to the name. + name = self.aliases()[name] + except KeyError: + pass + + for path in self.paths(): + if name in path: + return path[:path.index(name) + 1] + raise ValueError('Name should be defined in the import itself') + + def is_nested(self): + return False # By default, sub classes may overwrite this behavior + + def is_star_import(self): + return self.children[-1] == '*' + + def nodes_to_execute(self, last_added=False): + """ + `nodes_to_execute` works a bit different for imports, because the names + itself cannot directly get resolved (except on itself). + """ + # TODO couldn't we return the names? Would be nicer. + return [self] + + +class ImportFrom(Import): + type = 'import_from' + __slots__ = () + + def get_defined_names(self): + return [alias or name for name, alias in self._as_name_tuples()] + + def aliases(self): + """Mapping from alias to its corresponding name.""" + return dict((alias, name) for name, alias in self._as_name_tuples() + if alias is not None) + + def get_from_names(self): + for n in self.children[1:]: + if n not in ('.', '...'): + break + if n.type == 'dotted_name': # from x.y import + return n.children[::2] + elif n == 'import': # from . import + return [] + else: # from x import + return [n] + + @property + def level(self): + """The level parameter of ``__import__``.""" + level = 0 + for n in self.children[1:]: + if n in ('.', '...'): + level += len(n.value) + else: + break + return level + + def _as_name_tuples(self): + last = self.children[-1] + if last == ')': + last = self.children[-2] + elif last == '*': + return # No names defined directly. + + if last.type == 'import_as_names': + as_names = last.children[::2] + else: + as_names = [last] + for as_name in as_names: + if as_name.type == 'name': + yield as_name, None + else: + yield as_name.children[::2] # yields x, y -> ``x as y`` + + def star_import_name(self): + """ + The last name defined in a star import. + """ + return self.paths()[-1][-1] + + def paths(self): + """ + The import paths defined in an import statement. Typically an array + like this: ``[, ]``. + """ + dotted = self.get_from_names() + + if self.children[-1] == '*': + return [dotted] + return [dotted + [name] for name, alias in self._as_name_tuples()] + + +class ImportName(Import): + """For ``import_name`` nodes. Covers normal imports without ``from``.""" + type = 'import_name' + __slots__ = () + + def get_defined_names(self): + return [alias or path[0] for path, alias in self._dotted_as_names()] + + @property + def level(self): + """The level parameter of ``__import__``.""" + return 0 # Obviously 0 for imports without from. + + def paths(self): + return [path for path, alias in self._dotted_as_names()] + + def _dotted_as_names(self): + """Generator of (list(path), alias) where alias may be None.""" + dotted_as_names = self.children[1] + if dotted_as_names.type == 'dotted_as_names': + as_names = dotted_as_names.children[::2] + else: + as_names = [dotted_as_names] + + for as_name in as_names: + if as_name.type == 'dotted_as_name': + alias = as_name.children[2] + as_name = as_name.children[0] + else: + alias = None + if as_name.type == 'name': + yield [as_name], alias + else: + # dotted_names + yield as_name.children[::2], alias + + def is_nested(self): + """ + This checks for the special case of nested imports, without aliases and + from statement:: + + import foo.bar + """ + return [1 for path, alias in self._dotted_as_names() + if alias is None and len(path) > 1] + + def aliases(self): + return dict((alias, path[-1]) for path, alias in self._dotted_as_names() + if alias is not None) + + +class KeywordStatement(BaseNode): + """ + For the following statements: `assert`, `del`, `global`, `nonlocal`, + `raise`, `return`, `yield`, `return`, `yield`. + + `pass`, `continue` and `break` are not in there, because they are just + simple keywords and the parser reduces it to a keyword. + """ + __slots__ = () + + @property + def type(self): + """ + Keyword statements start with the keyword and end with `_stmt`. You can + crosscheck this with the Python grammar. + """ + return '%s_stmt' % self.keyword + + @property + def keyword(self): + return self.children[0].value + + def nodes_to_execute(self, last_added=False): + result = [] + for child in self.children: + result += child.nodes_to_execute() + return result + + +class AssertStmt(KeywordStatement): + __slots__ = () + + def assertion(self): + return self.children[1] + + +class GlobalStmt(KeywordStatement): + __slots__ = () + + def get_defined_names(self): + return [] + + def get_global_names(self): + return self.children[1::2] + + def nodes_to_execute(self, last_added=False): + """ + The global keyword allows to define any name. Even if it doesn't + exist. + """ + return [] + + +class ReturnStmt(KeywordStatement): + __slots__ = () + + +class YieldExpr(BaseNode): + __slots__ = () + + @property + def type(self): + return 'yield_expr' + + def nodes_to_execute(self, last_added=False): + if len(self.children) > 1: + return self.children[1].nodes_to_execute() + else: + return [] + + +def _defined_names(current): + """ + A helper function to find the defined names in statements, for loops and + list comprehensions. + """ + names = [] + if current.type in ('testlist_star_expr', 'testlist_comp', 'exprlist'): + for child in current.children[::2]: + names += _defined_names(child) + elif current.type in ('atom', 'star_expr'): + names += _defined_names(current.children[1]) + elif current.type in ('power', 'atom_expr'): + if current.children[-2] != '**': # Just if there's no operation + trailer = current.children[-1] + if trailer.children[0] == '.': + names.append(trailer.children[1]) + else: + names.append(current) + return names + + +class ExprStmt(BaseNode, DocstringMixin): + type = 'expr_stmt' + __slots__ = () + + def get_defined_names(self): + names = [] + if self.children[1].type == 'annassign': + names = _defined_names(self.children[0]) + return list(chain.from_iterable( + _defined_names(self.children[i]) + for i in range(0, len(self.children) - 2, 2) + if '=' in self.children[i + 1].value) + ) + names + + def get_rhs(self): + """Returns the right-hand-side of the equals.""" + return self.children[-1] + + def first_operation(self): + """ + Returns `+=`, `=`, etc or None if there is no operation. + """ + try: + return self.children[1] + except IndexError: + return None + + def nodes_to_execute(self, last_added=False): + # I think evaluating the statement (and possibly returned arrays), + # should be enough for static analysis. + result = [self] + for child in self.children: + result += child.nodes_to_execute(last_added=True) + return result + + +class Param(BaseNode): + """ + It's a helper class that makes business logic with params much easier. The + Python grammar defines no ``param`` node. It defines it in a different way + that is not really suited to working with parameters. + """ + type = 'param' + + def __init__(self, children, parent): + super(Param, self).__init__(children) + self.parent = parent + for child in children: + child.parent = self + + @property + def stars(self): + first = self.children[0] + if first in ('*', '**'): + return len(first.value) + return 0 + + @property + def default(self): + try: + return self.children[int(self.children[0] in ('*', '**')) + 2] + except IndexError: + return None + + def annotation(self): + tfpdef = self._tfpdef() + if tfpdef.type == 'tfpdef': + assert tfpdef.children[1] == ":" + assert len(tfpdef.children) == 3 + annotation = tfpdef.children[2] + return annotation + else: + return None + + def _tfpdef(self): + """ + tfpdef: see grammar.txt. + """ + offset = int(self.children[0] in ('*', '**')) + return self.children[offset] + + @property + def name(self): + if self._tfpdef().type == 'tfpdef': + return self._tfpdef().children[0] + else: + return self._tfpdef() + + @property + def position_nr(self): + return self.parent.children.index(self) - 1 + + def get_parent_function(self): + return search_ancestor(self, ('funcdef', 'lambda')) + + def __repr__(self): + default = '' if self.default is None else '=%s' % self.default.get_code() + return '<%s: %s>' % (type(self).__name__, str(self._tfpdef()) + default) + + def get_description(self): + children = self.children + if children[-1] == ',': + children = children[:-1] + return self._get_code_for_children(children, False, False) + + +class CompFor(BaseNode): + type = 'comp_for' + __slots__ = () + + def get_comp_fors(self): + yield self + last = self.children[-1] + while True: + if isinstance(last, CompFor): + yield last + elif not last.type == 'comp_if': + break + last = last.children[-1] + + def is_scope(self): + return True + + def get_defined_names(self): + return _defined_names(self.children[1]) + + def nodes_to_execute(self, last_added=False): + last = self.children[-1] + if last.type == 'comp_if': + for node in last.children[-1].nodes_to_execute(): + yield node + last = self.children[-2] + elif last.type == 'comp_for': + for node in last.nodes_to_execute(): + yield node + last = self.children[-2] + for node in last.nodes_to_execute(): + yield node diff --git a/jedi/parser/tree.py b/jedi/parser/tree.py index 7c7d6c2c..e69de29b 100644 --- a/jedi/parser/tree.py +++ b/jedi/parser/tree.py @@ -1,1632 +0,0 @@ -""" -If you know what an abstract syntax tree (AST) is, you'll see that this module -is pretty much that. The classes represent syntax elements like functions and -imports. - -This is the "business logic" part of the parser. There's a lot of logic here -that makes it easier for Jedi (and other libraries) to deal with a Python syntax -tree. - -By using `get_code` on a module, you can get back the 1-to-1 representation of -the input given to the parser. This is important if you are using refactoring. - -The easiest way to play with this module is to use :class:`parsing.Parser`. -:attr:`parsing.Parser.module` holds an instance of :class:`Module`: - ->>> from jedi._compatibility import u ->>> from jedi.parser.python import load_grammar ->>> from jedi.parser import ParserWithRecovery ->>> parser = ParserWithRecovery(load_grammar(), u('import os'), 'example.py') ->>> submodule = parser.module ->>> submodule - - -Any subclasses of :class:`Scope`, including :class:`Module` has an attribute -:attr:`imports `: - ->>> submodule.imports -[] - -See also :attr:`Scope.subscopes` and :attr:`Scope.statements`. - -For static analysis purposes there exists a method called -``nodes_to_execute`` on all nodes and leaves. It's documented in the static -anaylsis documentation. -""" -import os -import re -from inspect import cleandoc -from itertools import chain -import textwrap -import abc - -from jedi._compatibility import (Python3Method, encoding, is_py3, utf8_repr, - literal_eval, unicode) - - -def _safe_literal_eval(value): - first_two = value[:2].lower() - if first_two[0] == 'f' or first_two in ('fr', 'rf'): - # literal_eval is not able to resovle f literals. We have to do that - # manually in a later stage - return '' - - try: - return literal_eval(value) - except SyntaxError: - # It's possible to create syntax errors with literals like rb'' in - # Python 2. This should not be possible and in that case just return an - # empty string. - # Before Python 3.3 there was a more strict definition in which order - # you could define literals. - return '' - - -def search_ancestor(node, node_type_or_types): - if not isinstance(node_type_or_types, (list, tuple)): - node_type_or_types = (node_type_or_types,) - - while True: - node = node.parent - if node is None or node.type in node_type_or_types: - return node - - -class DocstringMixin(object): - __slots__ = () - - @property - def raw_doc(self): - """ Returns a cleaned version of the docstring token. """ - if isinstance(self, Module): - node = self.children[0] - elif isinstance(self, ClassOrFunc): - node = self.children[self.children.index(':') + 1] - if node.type == 'suite': # Normally a suite - node = node.children[1] # -> NEWLINE stmt - else: # ExprStmt - simple_stmt = self.parent - c = simple_stmt.parent.children - index = c.index(simple_stmt) - if not index: - return '' - node = c[index - 1] - - if node.type == 'simple_stmt': - node = node.children[0] - - if node.type == 'string': - # TODO We have to check next leaves until there are no new - # leaves anymore that might be part of the docstring. A - # docstring can also look like this: ``'foo' 'bar' - # Returns a literal cleaned version of the ``Token``. - cleaned = cleandoc(_safe_literal_eval(node.value)) - # Since we want the docstr output to be always unicode, just - # force it. - if is_py3 or isinstance(cleaned, unicode): - return cleaned - else: - return unicode(cleaned, 'UTF-8', 'replace') - return '' - - -class Base(object): - """ - This is just here to have an isinstance check, which is also used on - evaluate classes. But since they have sometimes a special type of - delegation, it is important for those classes to override this method. - - I know that there is a chance to do such things with __instancecheck__, but - since Python 2.5 doesn't support it, I decided to do it this way. - """ - __slots__ = () - - def get_root_node(self): - scope = self - while scope.parent is not None: - scope = scope.parent - return scope - - def get_parent_scope(self, include_flows=False): - """ - Returns the underlying scope. - """ - scope = self.parent - while scope is not None: - if include_flows and isinstance(scope, Flow): - return scope - if scope.is_scope(): - break - scope = scope.parent - return scope - - def get_definition(self): - if self.type in ('newline', 'endmarker'): - raise ValueError('Cannot get the indentation of whitespace or indentation.') - scope = self - while scope.parent is not None: - parent = scope.parent - if isinstance(scope, (Node, Leaf)) and parent.type != 'simple_stmt': - if scope.type == 'testlist_comp': - try: - if isinstance(scope.children[1], CompFor): - return scope.children[1] - except IndexError: - pass - scope = parent - else: - break - return scope - - def assignment_indexes(self): - """ - Returns an array of tuple(int, node) of the indexes that are used in - tuple assignments. - - For example if the name is ``y`` in the following code:: - - x, (y, z) = 2, '' - - would result in ``[(1, xyz_node), (0, yz_node)]``. - """ - indexes = [] - node = self.parent - compare = self - while node is not None: - if node.type in ('testlist_comp', 'testlist_star_expr', 'exprlist'): - for i, child in enumerate(node.children): - if child == compare: - indexes.insert(0, (int(i / 2), node)) - break - else: - raise LookupError("Couldn't find the assignment.") - elif isinstance(node, (ExprStmt, CompFor)): - break - - compare = node - node = node.parent - return indexes - - def is_scope(self): - # Default is not being a scope. Just inherit from Scope. - return False - - @abc.abstractmethod - def nodes_to_execute(self, last_added=False): - raise NotImplementedError() - - def get_next_sibling(self): - """ - The node immediately following the invocant in their parent's children - list. If the invocant does not have a next sibling, it is None - """ - # Can't use index(); we need to test by identity - for i, child in enumerate(self.parent.children): - if child is self: - try: - return self.parent.children[i + 1] - except IndexError: - return None - - def get_previous_sibling(self): - """ - The node/leaf immediately preceding the invocant in their parent's - children list. If the invocant does not have a previous sibling, it is - None. - """ - # Can't use index(); we need to test by identity - for i, child in enumerate(self.parent.children): - if child is self: - if i == 0: - return None - return self.parent.children[i - 1] - - def get_previous_leaf(self): - """ - Returns the previous leaf in the parser tree. - Raises an IndexError if it's the first element. - """ - node = self - while True: - c = node.parent.children - i = c.index(node) - if i == 0: - node = node.parent - if node.parent is None: - return None - else: - node = c[i - 1] - break - - while True: - try: - node = node.children[-1] - except AttributeError: # A Leaf doesn't have children. - return node - - def get_next_leaf(self): - """ - Returns the previous leaf in the parser tree. - Raises an IndexError if it's the last element. - """ - node = self - while True: - c = node.parent.children - i = c.index(node) - if i == len(c) - 1: - node = node.parent - if node.parent is None: - return None - else: - node = c[i + 1] - break - - while True: - try: - node = node.children[0] - except AttributeError: # A Leaf doesn't have children. - return node - - -class Leaf(Base): - __slots__ = ('value', 'parent', 'line', 'indent', 'prefix') - - def __init__(self, value, start_pos, prefix=''): - self.value = value - self.start_pos = start_pos - self.prefix = prefix - self.parent = None - - @property - def start_pos(self): - return self.line, self.indent - - @start_pos.setter - def start_pos(self, value): - self.line = value[0] - self.indent = value[1] - - def get_start_pos_of_prefix(self): - previous_leaf = self.get_previous_leaf() - if previous_leaf is None: - return self.line - self.prefix.count('\n'), 0 # It's the first leaf. - return previous_leaf.end_pos - - @property - def end_pos(self): - return self.line, self.indent + len(self.value) - - def move(self, line_offset): - self.line += line_offset - - def get_first_leaf(self): - return self - - def get_last_leaf(self): - return self - - def get_code(self, normalized=False, include_prefix=True): - if normalized: - return self.value - if include_prefix: - return self.prefix + self.value - else: - return self.value - - def nodes_to_execute(self, last_added=False): - return [] - - @utf8_repr - def __repr__(self): - return "<%s: %s start=%s>" % (type(self).__name__, self.value, self.start_pos) - - -class LeafWithNewLines(Leaf): - __slots__ = () - - @property - def end_pos(self): - """ - Literals and whitespace end_pos are more complicated than normal - end_pos, because the containing newlines may change the indexes. - """ - lines = self.value.split('\n') - end_pos_line = self.line + len(lines) - 1 - # Check for multiline token - if self.line == end_pos_line: - end_pos_indent = self.indent + len(lines[-1]) - else: - end_pos_indent = len(lines[-1]) - return end_pos_line, end_pos_indent - - @utf8_repr - def __repr__(self): - return "<%s: %r>" % (type(self).__name__, self.value) - - -class EndMarker(Leaf): - __slots__ = () - type = 'endmarker' - - -class Newline(LeafWithNewLines): - """Contains NEWLINE and ENDMARKER tokens.""" - __slots__ = () - type = 'newline' - - @utf8_repr - def __repr__(self): - return "<%s: %s>" % (type(self).__name__, repr(self.value)) - - -class Name(Leaf): - """ - A string. Sometimes it is important to know if the string belongs to a name - or not. - """ - type = 'name' - __slots__ = () - - def __str__(self): - return self.value - - def __unicode__(self): - return self.value - - def __repr__(self): - return "<%s: %s@%s,%s>" % (type(self).__name__, self.value, - self.line, self.indent) - - def is_definition(self): - if self.parent.type in ('power', 'atom_expr'): - # In `self.x = 3` self is not a definition, but x is. - return False - - stmt = self.get_definition() - if stmt.type in ('funcdef', 'classdef', 'file_input', 'param'): - return self == stmt.name - elif stmt.type == 'for_stmt': - return self.start_pos < stmt.children[2].start_pos - elif stmt.type == 'try_stmt': - return self.get_previous_sibling() == 'as' - else: - return stmt.type in ('expr_stmt', 'import_name', 'import_from', - 'comp_for', 'with_stmt') \ - and self in stmt.get_defined_names() - - def nodes_to_execute(self, last_added=False): - if last_added is False: - yield self - - -class Literal(LeafWithNewLines): - __slots__ = () - - def eval(self): - return _safe_literal_eval(self.value) - - -class Number(Literal): - type = 'number' - __slots__ = () - - -class String(Literal): - type = 'string' - __slots__ = () - - -class Operator(Leaf): - type = 'operator' - __slots__ = () - - def __str__(self): - return self.value - - def __eq__(self, other): - """ - Make comparisons with strings easy. - Improves the readability of the parser. - """ - if isinstance(other, Operator): - return self is other - else: - return self.value == other - - def __ne__(self, other): - """Python 2 compatibility.""" - return self.value != other - - def __hash__(self): - return hash(self.value) - - -class Keyword(Leaf): - type = 'keyword' - __slots__ = () - - def __eq__(self, other): - """ - Make comparisons with strings easy. - Improves the readability of the parser. - """ - if isinstance(other, Keyword): - return self is other - return self.value == other - - def __ne__(self, other): - """Python 2 compatibility.""" - return not self.__eq__(other) - - def __hash__(self): - return hash(self.value) - - -class BaseNode(Base): - """ - The super class for Scope, Import, Name and Statement. Every object in - the parser tree inherits from this class. - """ - __slots__ = ('children', 'parent') - type = None - - def __init__(self, children): - """ - Initialize :class:`BaseNode`. - - :param children: The module in which this Python object locates. - """ - for c in children: - c.parent = self - self.children = children - self.parent = None - - def move(self, line_offset): - """ - Move the Node's start_pos. - """ - for c in self.children: - c.move(line_offset) - - @property - def start_pos(self): - return self.children[0].start_pos - - def get_start_pos_of_prefix(self): - return self.children[0].get_start_pos_of_prefix() - - @property - def end_pos(self): - return self.children[-1].end_pos - - def _get_code_for_children(self, children, normalized, include_prefix): - # TODO implement normalized (depending on context). - if include_prefix: - return "".join(c.get_code(normalized) for c in children) - else: - first = children[0].get_code(include_prefix=False) - return first + "".join(c.get_code(normalized) for c in children[1:]) - - def get_code(self, normalized=False, include_prefix=True): - return self._get_code_for_children(self.children, normalized, include_prefix) - - @Python3Method - def name_for_position(self, position): - for c in self.children: - if isinstance(c, Leaf): - if isinstance(c, Name) and c.start_pos <= position <= c.end_pos: - return c - else: - result = c.name_for_position(position) - if result is not None: - return result - return None - - def get_leaf_for_position(self, position, include_prefixes=False): - def binary_search(lower, upper): - if lower == upper: - element = self.children[lower] - if not include_prefixes and position < element.start_pos: - # We're on a prefix. - return None - # In case we have prefixes, a leaf always matches - try: - return element.get_leaf_for_position(position, include_prefixes) - except AttributeError: - return element - - - index = int((lower + upper) / 2) - element = self.children[index] - if position <= element.end_pos: - return binary_search(lower, index) - else: - return binary_search(index + 1, upper) - - if not ((1, 0) <= position <= self.children[-1].end_pos): - raise ValueError('Please provide a position that exists within this node.') - return binary_search(0, len(self.children) - 1) - - @Python3Method - def get_statement_for_position(self, pos): - for c in self.children: - if c.start_pos <= pos <= c.end_pos: - if c.type not in ('decorated', 'simple_stmt', 'suite') \ - and not isinstance(c, (Flow, ClassOrFunc)): - return c - else: - try: - return c.get_statement_for_position(pos) - except AttributeError: - pass # Must be a non-scope - return None - - def get_first_leaf(self): - return self.children[0].get_first_leaf() - - def get_last_leaf(self): - return self.children[-1].get_last_leaf() - - def get_following_comment_same_line(self): - """ - returns (as string) any comment that appears on the same line, - after the node, including the # - """ - try: - if self.type == 'for_stmt': - whitespace = self.children[5].get_first_leaf().prefix - elif self.type == 'with_stmt': - whitespace = self.children[3].get_first_leaf().prefix - else: - whitespace = self.get_last_leaf().get_next_leaf().prefix - except AttributeError: - return None - except ValueError: - # TODO in some particular cases, the tree doesn't seem to be linked - # correctly - return None - if "#" not in whitespace: - return None - comment = whitespace[whitespace.index("#"):] - if "\r" in comment: - comment = comment[:comment.index("\r")] - if "\n" in comment: - comment = comment[:comment.index("\n")] - return comment - - @utf8_repr - def __repr__(self): - code = self.get_code().replace('\n', ' ').strip() - if not is_py3: - code = code.encode(encoding, 'replace') - return "<%s: %s@%s,%s>" % \ - (type(self).__name__, code, self.start_pos[0], self.start_pos[1]) - - -class Node(BaseNode): - """Concrete implementation for interior nodes.""" - __slots__ = ('type',) - - _IGNORE_EXECUTE_NODES = set([ - 'suite', 'subscriptlist', 'subscript', 'simple_stmt', 'sliceop', - 'testlist_comp', 'dictorsetmaker', 'trailer', 'decorators', - 'decorated', 'arglist', 'argument', 'exprlist', 'testlist', - 'testlist_safe', 'testlist1' - ]) - - def __init__(self, type, children): - """ - Initializer. - - Takes a type constant (a symbol number >= 256), a sequence of - child nodes, and an optional context keyword argument. - - As a side effect, the parent pointers of the children are updated. - """ - super(Node, self).__init__(children) - self.type = type - - def nodes_to_execute(self, last_added=False): - """ - For static analysis. - """ - result = [] - if self.type not in Node._IGNORE_EXECUTE_NODES and not last_added: - result.append(self) - last_added = True - - for child in self.children: - result += child.nodes_to_execute(last_added) - return result - - def __repr__(self): - return "%s(%s, %r)" % (self.__class__.__name__, self.type, self.children) - - -class ErrorNode(BaseNode): - """ - TODO doc - """ - __slots__ = () - type = 'error_node' - - def nodes_to_execute(self, last_added=False): - return [] - - -class ErrorLeaf(LeafWithNewLines): - """ - TODO doc - """ - __slots__ = ('original_type') - type = 'error_leaf' - - def __init__(self, original_type, value, start_pos, prefix=''): - super(ErrorLeaf, self).__init__(value, start_pos, prefix) - self.original_type = original_type - - def __repr__(self): - return "<%s: %s:%s, %s)>" % \ - (type(self).__name__, self.original_type, repr(self.value), self.start_pos) - - -class Scope(BaseNode, DocstringMixin): - """ - Super class for the parser tree, which represents the state of a python - text file. - A Scope manages and owns its subscopes, which are classes and functions, as - well as variables and imports. It is used to access the structure of python - files. - - :param start_pos: The position (line and column) of the scope. - :type start_pos: tuple(int, int) - """ - __slots__ = () - - def __init__(self, children): - super(Scope, self).__init__(children) - - @property - def returns(self): - # Needed here for fast_parser, because the fast_parser splits and - # returns will be in "normal" modules. - return self._search_in_scope(ReturnStmt) - - @property - def subscopes(self): - return self._search_in_scope(Scope) - - @property - def flows(self): - return self._search_in_scope(Flow) - - @property - def imports(self): - return self._search_in_scope(Import) - - @Python3Method - def _search_in_scope(self, typ): - def scan(children): - elements = [] - for element in children: - if isinstance(element, typ): - elements.append(element) - if element.type in ('suite', 'simple_stmt', 'decorated') \ - or isinstance(element, Flow): - elements += scan(element.children) - return elements - - return scan(self.children) - - @property - def statements(self): - return self._search_in_scope((ExprStmt, KeywordStatement)) - - def is_scope(self): - return True - - def __repr__(self): - try: - name = self.path - except AttributeError: - try: - name = self.name - except AttributeError: - name = self.command - - return "<%s: %s@%s-%s>" % (type(self).__name__, name, - self.start_pos[0], self.end_pos[0]) - - def walk(self): - yield self - for s in self.subscopes: - for scope in s.walk(): - yield scope - - for r in self.statements: - while isinstance(r, Flow): - for scope in r.walk(): - yield scope - r = r.next - - -class Module(Scope): - """ - The top scope, which is always a module. - Depending on the underlying parser this may be a full module or just a part - of a module. - """ - __slots__ = ('path', 'used_names', '_name') - type = 'file_input' - - def __init__(self, children): - """ - Initialize :class:`Module`. - - :type path: str - :arg path: File path to this module. - - .. todo:: Document `top_module`. - """ - super(Module, self).__init__(children) - self.path = None # Set later. - - @property - def name(self): - """ This is used for the goto functions. """ - if self.path is None: - string = '' # no path -> empty name - else: - sep = (re.escape(os.path.sep),) * 2 - r = re.search(r'([^%s]*?)(%s__init__)?(\.py|\.so)?$' % sep, self.path) - # Remove PEP 3149 names - string = re.sub('\.[a-z]+-\d{2}[mud]{0,3}$', '', r.group(1)) - # Positions are not real, but a module starts at (1, 0) - p = (1, 0) - name = Name(string, p) - name.parent = self - return name - - @property - def has_explicit_absolute_import(self): - """ - Checks if imports in this module are explicitly absolute, i.e. there - is a ``__future__`` import. - """ - # TODO this is a strange scan and not fully correct. I think Python's - # parser does it in a different way and scans for the first - # statement/import with a tokenizer (to check for syntax changes like - # the future print statement). - for imp in self.imports: - if imp.type == 'import_from' and imp.level == 0: - for path in imp.paths(): - if [str(name) for name in path] == ['__future__', 'absolute_import']: - return True - return False - - def nodes_to_execute(self, last_added=False): - # Yield itself, class needs to be executed for decorator checks. - result = [] - for child in self.children: - result += child.nodes_to_execute() - return result - - -class Decorator(BaseNode): - type = 'decorator' - __slots__ = () - - def nodes_to_execute(self, last_added=False): - if self.children[-2] == ')': - node = self.children[-3] - if node != '(': - return node.nodes_to_execute() - return [] - - -class ClassOrFunc(Scope): - __slots__ = () - - @property - def name(self): - return self.children[1] - - def get_decorators(self): - decorated = self.parent - if decorated.type == 'decorated': - if decorated.children[0].type == 'decorators': - return decorated.children[0].children - else: - return decorated.children[:1] - else: - return [] - - -class Class(ClassOrFunc): - """ - Used to store the parsed contents of a python class. - - :param name: The Class name. - :type name: str - :param supers: The super classes of a Class. - :type supers: list - :param start_pos: The start position (line, column) of the class. - :type start_pos: tuple(int, int) - """ - type = 'classdef' - __slots__ = () - - def __init__(self, children): - super(Class, self).__init__(children) - - def get_super_arglist(self): - if self.children[2] != '(': # Has no parentheses - return None - else: - if self.children[3] == ')': # Empty parentheses - return None - else: - return self.children[3] - - @property - def doc(self): - """ - Return a document string including call signature of __init__. - """ - docstr = self.raw_doc - for sub in self.subscopes: - if str(sub.name) == '__init__': - return '%s\n\n%s' % ( - sub.get_call_signature(func_name=self.name), docstr) - return docstr - - def nodes_to_execute(self, last_added=False): - # Yield itself, class needs to be executed for decorator checks. - yield self - # Super arguments. - arglist = self.get_super_arglist() - try: - children = arglist.children - except AttributeError: - if arglist is not None: - for node_to_execute in arglist.nodes_to_execute(): - yield node_to_execute - else: - for argument in children: - if argument.type == 'argument': - # metaclass= or list comprehension or */** - raise NotImplementedError('Metaclasses not implemented') - else: - for node_to_execute in argument.nodes_to_execute(): - yield node_to_execute - - # care for the class suite: - for node in self.children[self.children.index(':'):]: - # This could be easier without the fast parser. But we need to find - # the position of the colon, because everything after it can be a - # part of the class, not just its suite. - for node_to_execute in node.nodes_to_execute(): - yield node_to_execute - - -def _create_params(parent, argslist_list): - """ - `argslist_list` is a list that can contain an argslist as a first item, but - most not. It's basically the items between the parameter brackets (which is - at most one item). - This function modifies the parser structure. It generates `Param` objects - from the normal ast. Those param objects do not exist in a normal ast, but - make the evaluation of the ast tree so much easier. - You could also say that this function replaces the argslist node with a - list of Param objects. - """ - def check_python2_nested_param(node): - """ - Python 2 allows params to look like ``def x(a, (b, c))``, which is - basically a way of unpacking tuples in params. Python 3 has ditched - this behavior. Jedi currently just ignores those constructs. - """ - return node.type == 'tfpdef' and node.children[0] == '(' - - try: - first = argslist_list[0] - except IndexError: - return [] - - if first.type in ('name', 'tfpdef'): - if check_python2_nested_param(first): - return [first] - else: - return [Param([first], parent)] - elif first == '*': - return [first] - else: # argslist is a `typedargslist` or a `varargslist`. - children = first.children - new_children = [] - start = 0 - # Start with offset 1, because the end is higher. - for end, child in enumerate(children + [None], 1): - if child is None or child == ',': - param_children = children[start:end] - if param_children: # Could as well be comma and then end. - if check_python2_nested_param(param_children[0]): - new_children += param_children - elif param_children[0] == '*' and param_children[1] == ',': - new_children += param_children - else: - new_children.append(Param(param_children, parent)) - start = end - return new_children - - -class Function(ClassOrFunc): - """ - Used to store the parsed contents of a python function. - - Children: - 0) - 1) - 2) parameter list (including open-paren and close-paren s) - 3 or 5) - 4 or 6) Node() representing function body - 3) -> (if annotation is also present) - 4) annotation (if present) - """ - type = 'funcdef' - - def __init__(self, children): - super(Function, self).__init__(children) - parameters = self.children[2] # After `def foo` - parameters.children[1:-1] = _create_params(parameters, parameters.children[1:-1]) - - @property - def params(self): - return [p for p in self.children[2].children if p.type == 'param'] - - @property - def name(self): - return self.children[1] # First token after `def` - - @property - def yields(self): - # TODO This is incorrect, yields are also possible in a statement. - return self._search_in_scope(YieldExpr) - - def is_generator(self): - return bool(self.yields) - - def annotation(self): - try: - if self.children[3] == "->": - return self.children[4] - assert self.children[3] == ":" - return None - except IndexError: - return None - - def get_call_signature(self, width=72, func_name=None): - """ - Generate call signature of this function. - - :param width: Fold lines if a line is longer than this value. - :type width: int - :arg func_name: Override function name when given. - :type func_name: str - - :rtype: str - """ - func_name = func_name or self.name - code = unicode(func_name) + self._get_paramlist_code() - return '\n'.join(textwrap.wrap(code, width)) - - def _get_paramlist_code(self): - return self.children[2].get_code() - - @property - def doc(self): - """ Return a document string including call signature. """ - docstr = self.raw_doc - return '%s\n\n%s' % (self.get_call_signature(), docstr) - - def nodes_to_execute(self, last_added=False): - # Yield itself, functions needs to be executed for decorator checks. - yield self - for param in self.params: - if param.default is not None: - yield param.default - # care for the function suite: - for node in self.children[4:]: - # This could be easier without the fast parser. The fast parser - # allows that the 4th position is empty or that there's even a - # fifth element (another function/class). So just scan everything - # after colon. - for node_to_execute in node.nodes_to_execute(): - yield node_to_execute - - -class Lambda(Function): - """ - Lambdas are basically trimmed functions, so give it the same interface. - - Children: - 0) - *) for each argument x - -2) - -1) Node() representing body - """ - type = 'lambda' - __slots__ = () - - def __init__(self, children): - # We don't want to call the Function constructor, call its parent. - super(Function, self).__init__(children) - lst = self.children[1:-2] # Everything between `lambda` and the `:` operator is a parameter. - self.children[1:-2] = _create_params(self, lst) - - @property - def name(self): - # Borrow the position of the AST node. - return Name('', self.children[0].start_pos) - - def _get_paramlist_code(self): - return '(' + ''.join(param.get_code() for param in self.params).strip() + ')' - - @property - def params(self): - return self.children[1:-2] - - def is_generator(self): - return False - - def annotation(self): - # lambda functions do not support annotations - return None - - @property - def yields(self): - return [] - - def nodes_to_execute(self, last_added=False): - for param in self.params: - if param.default is not None: - yield param.default - # Care for the lambda test (last child): - for node_to_execute in self.children[-1].nodes_to_execute(): - yield node_to_execute - - def __repr__(self): - return "<%s@%s>" % (self.__class__.__name__, self.start_pos) - - -class Flow(BaseNode): - __slots__ = () - FLOW_KEYWORDS = ( - 'try', 'except', 'finally', 'else', 'if', 'elif', 'with', 'for', 'while' - ) - - def nodes_to_execute(self, last_added=False): - for child in self.children: - for node_to_execute in child.nodes_to_execute(): - yield node_to_execute - - def get_branch_keyword(self, node): - start_pos = node.start_pos - if not (self.start_pos < start_pos <= self.end_pos): - raise ValueError('The node is not part of the flow.') - - keyword = None - for i, child in enumerate(self.children): - if start_pos < child.start_pos: - return keyword - first_leaf = child.get_first_leaf() - if first_leaf in self.FLOW_KEYWORDS: - keyword = first_leaf - return 0 - - -class IfStmt(Flow): - type = 'if_stmt' - __slots__ = () - - def check_nodes(self): - """ - Returns all the `test` nodes that are defined as x, here: - - if x: - pass - elif x: - pass - """ - for i, c in enumerate(self.children): - if c in ('elif', 'if'): - yield self.children[i + 1] - - def node_in_which_check_node(self, node): - """ - Returns the check node (see function above) that a node is contained - in. However if it the node is in the check node itself and not in the - suite return None. - """ - start_pos = node.start_pos - for check_node in reversed(list(self.check_nodes())): - if check_node.start_pos < start_pos: - if start_pos < check_node.end_pos: - return None - # In this case the node is within the check_node itself, - # not in the suite - else: - return check_node - - def node_after_else(self, node): - """ - Checks if a node is defined after `else`. - """ - for c in self.children: - if c == 'else': - if node.start_pos > c.start_pos: - return True - else: - return False - - -class WhileStmt(Flow): - type = 'while_stmt' - __slots__ = () - - -class ForStmt(Flow): - type = 'for_stmt' - __slots__ = () - - def get_input_node(self): - """ - Returns the input node ``y`` from: ``for x in y:``. - """ - return self.children[3] - - def defines_one_name(self): - """ - Returns True if only one name is returned: ``for x in y``. - Returns False if the for loop is more complicated: ``for x, z in y``. - - :returns: bool - """ - return self.children[1].type == 'name' - - -class TryStmt(Flow): - type = 'try_stmt' - __slots__ = () - - def except_clauses(self): - """ - Returns the ``test`` nodes found in ``except_clause`` nodes. - Returns ``[None]`` for except clauses without an exception given. - """ - for node in self.children: - if node.type == 'except_clause': - yield node.children[1] - elif node == 'except': - yield None - - def nodes_to_execute(self, last_added=False): - result = [] - for child in self.children[2::3]: - result += child.nodes_to_execute() - for child in self.children[0::3]: - if child.type == 'except_clause': - # Add the test node and ignore the `as NAME` definition. - result += child.children[1].nodes_to_execute() - return result - - -class WithStmt(Flow): - type = 'with_stmt' - __slots__ = () - - def get_defined_names(self): - names = [] - for with_item in self.children[1:-2:2]: - # Check with items for 'as' names. - if with_item.type == 'with_item': - names += _defined_names(with_item.children[2]) - return names - - def node_from_name(self, name): - node = name - while True: - node = node.parent - if node.type == 'with_item': - return node.children[0] - - def nodes_to_execute(self, last_added=False): - result = [] - for child in self.children[1::2]: - if child.type == 'with_item': - # Just ignore the `as EXPR` part - at least for now, because - # most times it's just a name. - child = child.children[0] - result += child.nodes_to_execute() - return result - - -class Import(BaseNode): - __slots__ = () - - def path_for_name(self, name): - try: - # The name may be an alias. If it is, just map it back to the name. - name = self.aliases()[name] - except KeyError: - pass - - for path in self.paths(): - if name in path: - return path[:path.index(name) + 1] - raise ValueError('Name should be defined in the import itself') - - def is_nested(self): - return False # By default, sub classes may overwrite this behavior - - def is_star_import(self): - return self.children[-1] == '*' - - def nodes_to_execute(self, last_added=False): - """ - `nodes_to_execute` works a bit different for imports, because the names - itself cannot directly get resolved (except on itself). - """ - # TODO couldn't we return the names? Would be nicer. - return [self] - - -class ImportFrom(Import): - type = 'import_from' - __slots__ = () - - def get_defined_names(self): - return [alias or name for name, alias in self._as_name_tuples()] - - def aliases(self): - """Mapping from alias to its corresponding name.""" - return dict((alias, name) for name, alias in self._as_name_tuples() - if alias is not None) - - def get_from_names(self): - for n in self.children[1:]: - if n not in ('.', '...'): - break - if n.type == 'dotted_name': # from x.y import - return n.children[::2] - elif n == 'import': # from . import - return [] - else: # from x import - return [n] - - @property - def level(self): - """The level parameter of ``__import__``.""" - level = 0 - for n in self.children[1:]: - if n in ('.', '...'): - level += len(n.value) - else: - break - return level - - def _as_name_tuples(self): - last = self.children[-1] - if last == ')': - last = self.children[-2] - elif last == '*': - return # No names defined directly. - - if last.type == 'import_as_names': - as_names = last.children[::2] - else: - as_names = [last] - for as_name in as_names: - if as_name.type == 'name': - yield as_name, None - else: - yield as_name.children[::2] # yields x, y -> ``x as y`` - - def star_import_name(self): - """ - The last name defined in a star import. - """ - return self.paths()[-1][-1] - - def paths(self): - """ - The import paths defined in an import statement. Typically an array - like this: ``[, ]``. - """ - dotted = self.get_from_names() - - if self.children[-1] == '*': - return [dotted] - return [dotted + [name] for name, alias in self._as_name_tuples()] - - -class ImportName(Import): - """For ``import_name`` nodes. Covers normal imports without ``from``.""" - type = 'import_name' - __slots__ = () - - def get_defined_names(self): - return [alias or path[0] for path, alias in self._dotted_as_names()] - - @property - def level(self): - """The level parameter of ``__import__``.""" - return 0 # Obviously 0 for imports without from. - - def paths(self): - return [path for path, alias in self._dotted_as_names()] - - def _dotted_as_names(self): - """Generator of (list(path), alias) where alias may be None.""" - dotted_as_names = self.children[1] - if dotted_as_names.type == 'dotted_as_names': - as_names = dotted_as_names.children[::2] - else: - as_names = [dotted_as_names] - - for as_name in as_names: - if as_name.type == 'dotted_as_name': - alias = as_name.children[2] - as_name = as_name.children[0] - else: - alias = None - if as_name.type == 'name': - yield [as_name], alias - else: - # dotted_names - yield as_name.children[::2], alias - - def is_nested(self): - """ - This checks for the special case of nested imports, without aliases and - from statement:: - - import foo.bar - """ - return [1 for path, alias in self._dotted_as_names() - if alias is None and len(path) > 1] - - def aliases(self): - return dict((alias, path[-1]) for path, alias in self._dotted_as_names() - if alias is not None) - - -class KeywordStatement(BaseNode): - """ - For the following statements: `assert`, `del`, `global`, `nonlocal`, - `raise`, `return`, `yield`, `return`, `yield`. - - `pass`, `continue` and `break` are not in there, because they are just - simple keywords and the parser reduces it to a keyword. - """ - __slots__ = () - - @property - def type(self): - """ - Keyword statements start with the keyword and end with `_stmt`. You can - crosscheck this with the Python grammar. - """ - return '%s_stmt' % self.keyword - - @property - def keyword(self): - return self.children[0].value - - def nodes_to_execute(self, last_added=False): - result = [] - for child in self.children: - result += child.nodes_to_execute() - return result - - -class AssertStmt(KeywordStatement): - __slots__ = () - - def assertion(self): - return self.children[1] - - -class GlobalStmt(KeywordStatement): - __slots__ = () - - def get_defined_names(self): - return [] - - def get_global_names(self): - return self.children[1::2] - - def nodes_to_execute(self, last_added=False): - """ - The global keyword allows to define any name. Even if it doesn't - exist. - """ - return [] - - -class ReturnStmt(KeywordStatement): - __slots__ = () - - -class YieldExpr(BaseNode): - __slots__ = () - - @property - def type(self): - return 'yield_expr' - - def nodes_to_execute(self, last_added=False): - if len(self.children) > 1: - return self.children[1].nodes_to_execute() - else: - return [] - - -def _defined_names(current): - """ - A helper function to find the defined names in statements, for loops and - list comprehensions. - """ - names = [] - if current.type in ('testlist_star_expr', 'testlist_comp', 'exprlist'): - for child in current.children[::2]: - names += _defined_names(child) - elif current.type in ('atom', 'star_expr'): - names += _defined_names(current.children[1]) - elif current.type in ('power', 'atom_expr'): - if current.children[-2] != '**': # Just if there's no operation - trailer = current.children[-1] - if trailer.children[0] == '.': - names.append(trailer.children[1]) - else: - names.append(current) - return names - - -class ExprStmt(BaseNode, DocstringMixin): - type = 'expr_stmt' - __slots__ = () - - def get_defined_names(self): - names = [] - if self.children[1].type == 'annassign': - names = _defined_names(self.children[0]) - return list(chain.from_iterable( - _defined_names(self.children[i]) - for i in range(0, len(self.children) - 2, 2) - if '=' in self.children[i + 1].value) - ) + names - - def get_rhs(self): - """Returns the right-hand-side of the equals.""" - return self.children[-1] - - def first_operation(self): - """ - Returns `+=`, `=`, etc or None if there is no operation. - """ - try: - return self.children[1] - except IndexError: - return None - - def nodes_to_execute(self, last_added=False): - # I think evaluating the statement (and possibly returned arrays), - # should be enough for static analysis. - result = [self] - for child in self.children: - result += child.nodes_to_execute(last_added=True) - return result - - -class Param(BaseNode): - """ - It's a helper class that makes business logic with params much easier. The - Python grammar defines no ``param`` node. It defines it in a different way - that is not really suited to working with parameters. - """ - type = 'param' - - def __init__(self, children, parent): - super(Param, self).__init__(children) - self.parent = parent - for child in children: - child.parent = self - - @property - def stars(self): - first = self.children[0] - if first in ('*', '**'): - return len(first.value) - return 0 - - @property - def default(self): - try: - return self.children[int(self.children[0] in ('*', '**')) + 2] - except IndexError: - return None - - def annotation(self): - tfpdef = self._tfpdef() - if tfpdef.type == 'tfpdef': - assert tfpdef.children[1] == ":" - assert len(tfpdef.children) == 3 - annotation = tfpdef.children[2] - return annotation - else: - return None - - def _tfpdef(self): - """ - tfpdef: see grammar.txt. - """ - offset = int(self.children[0] in ('*', '**')) - return self.children[offset] - - @property - def name(self): - if self._tfpdef().type == 'tfpdef': - return self._tfpdef().children[0] - else: - return self._tfpdef() - - @property - def position_nr(self): - return self.parent.children.index(self) - 1 - - def get_parent_function(self): - return search_ancestor(self, ('funcdef', 'lambda')) - - def __repr__(self): - default = '' if self.default is None else '=%s' % self.default.get_code() - return '<%s: %s>' % (type(self).__name__, str(self._tfpdef()) + default) - - def get_description(self): - children = self.children - if children[-1] == ',': - children = children[:-1] - return self._get_code_for_children(children, False, False) - - -class CompFor(BaseNode): - type = 'comp_for' - __slots__ = () - - def get_comp_fors(self): - yield self - last = self.children[-1] - while True: - if isinstance(last, CompFor): - yield last - elif not last.type == 'comp_if': - break - last = last.children[-1] - - def is_scope(self): - return True - - def get_defined_names(self): - return _defined_names(self.children[1]) - - def nodes_to_execute(self, last_added=False): - last = self.children[-1] - if last.type == 'comp_if': - for node in last.children[-1].nodes_to_execute(): - yield node - last = self.children[-2] - elif last.type == 'comp_for': - for node in last.nodes_to_execute(): - yield node - last = self.children[-2] - for node in last.nodes_to_execute(): - yield node diff --git a/jedi/refactoring.py b/jedi/refactoring.py index a342f08e..1e572638 100644 --- a/jedi/refactoring.py +++ b/jedi/refactoring.py @@ -16,7 +16,7 @@ import difflib from jedi import common from jedi.evaluate import helpers -from jedi.parser import tree as pt +from jedi.parser.python import tree as pt class Refactoring(object): diff --git a/test/test_parser/test_parser.py b/test/test_parser/test_parser.py index aec48808..cbf7beb1 100644 --- a/test/test_parser/test_parser.py +++ b/test/test_parser/test_parser.py @@ -6,7 +6,7 @@ import jedi from jedi._compatibility import u, is_py3 from jedi.parser import ParserWithRecovery from jedi.parser.python import parse, load_grammar -from jedi.parser import tree as pt +from jedi.parser.python import tree def test_user_statement_on_import(): @@ -17,7 +17,7 @@ def test_user_statement_on_import(): for pos in [(2, 1), (2, 4)]: p = parse(s) stmt = p.get_statement_for_position(pos) - assert isinstance(stmt, pt.Import) + assert isinstance(stmt, tree.Import) assert [str(n) for n in stmt.get_defined_names()] == ['time'] @@ -41,19 +41,19 @@ class TestCallAndName(): def test_call_type(self): call = self.get_call('hello') - assert isinstance(call, pt.Name) + assert isinstance(call, tree.Name) def test_literal_type(self): literal = self.get_call('1.0') - assert isinstance(literal, pt.Literal) + assert isinstance(literal, tree.Literal) assert type(literal.eval()) == float literal = self.get_call('1') - assert isinstance(literal, pt.Literal) + assert isinstance(literal, tree.Literal) assert type(literal.eval()) == int literal = self.get_call('"hello"') - assert isinstance(literal, pt.Literal) + assert isinstance(literal, tree.Literal) assert literal.eval() == 'hello' @@ -204,7 +204,7 @@ def test_param_splitting(): def test_unicode_string(): - s = pt.String(None, u('bö'), (0, 0)) + s = tree.String(None, u('bö'), (0, 0)) assert repr(s) # Should not raise an Error! diff --git a/test/test_parser/test_parser_tree.py b/test/test_parser/test_parser_tree.py index 9338cebb..57e0c640 100644 --- a/test/test_parser/test_parser_tree.py +++ b/test/test_parser/test_parser_tree.py @@ -6,7 +6,7 @@ import pytest from jedi._compatibility import u, unicode from jedi.parser.python import parse -from jedi.parser import tree as pt +from jedi.parser.python import tree class TestsFunctionAndLambdaParsing(object): @@ -34,14 +34,14 @@ class TestsFunctionAndLambdaParsing(object): @pytest.fixture() def expected(self, request, node): return request.keywords['expected'] - + def test_name(self, node, expected): - assert isinstance(node.name, pt.Name) + assert isinstance(node.name, tree.Name) assert unicode(node.name) == u(expected['name']) - + def test_params(self, node, expected): assert isinstance(node.params, list) - assert all(isinstance(x, pt.Param) for x in node.params) + assert all(isinstance(x, tree.Param) for x in node.params) assert [unicode(x.name) for x in node.params] == [u(x) for x in expected['params']] def test_is_generator(self, node, expected):