VimPlug/jedi-fork
1
0
Fork 0
forked from VimPlug/jedi
Code Pull Requests Activity

Merge branch 'dev' into unicode_tokenize_fix2

Browse Source 
Conflicts:
	AUTHORS.txt
This commit is contained in:
farhad
2015-03-06 12:14:38 +04:00
parent f9c104348e a6c5d9f0a6
commit 32081bd156
105 changed files with 7728 additions and 6520 deletions
Download Patch File Download Diff File Expand all files Collapse all files
jedi/parser/__init__.py
+340 -598
View File
@@ -1,7 +1,7 @@
"""
The ``Parser`` tries to convert the available Python code in an easy to read
format, something like an abstract syntax tree. The classes who represent this
tree, are sitting in the :mod:`jedi.parser.representation` module.
tree, are sitting in the :mod:`jedi.parser.tree` module.
The Python module ``tokenize`` is a very important part in the ``Parser``,
because it splits the code into different words (tokens). Sometimes it looks a
@@ -15,13 +15,16 @@ within the statement. This lowers memory usage and cpu time and reduces the
complexity of the ``Parser`` (there's another parser sitting inside
``Statement``, which produces ``Array`` and ``Call``).
"""
import keyword
import os
import re
from jedi._compatibility import next, unicode
from jedi import debug
from jedi import common
from jedi.parser import representation as pr
from jedi.parser import tree as pt
from jedi.parser import tokenize
from jedi.parser import token
from jedi.parser.token import (DEDENT, INDENT, ENDMARKER, NEWLINE, NUMBER,
STRING, OP, ERRORTOKEN)
from jedi.parser.pgen2.pgen import generate_grammar
from jedi.parser.pgen2.parse import PgenParser
OPERATOR_KEYWORDS = 'and', 'for', 'if', 'else', 'in', 'is', 'lambda', 'not', 'or'
# Not used yet. In the future I intend to add something like KeywordStatement
@@ -29,623 +32,362 @@ STATEMENT_KEYWORDS = 'assert', 'del', 'global', 'nonlocal', 'raise', \
'return', 'yield', 'pass', 'continue', 'break'
_loaded_grammars = {}
def load_grammar(file='grammar3.4'):
# For now we only support two different Python syntax versions: The latest
# Python 3 and Python 2. This may change.
if file.startswith('grammar3'):
file = 'grammar3.4'
else:
file = 'grammar2.7'
global _loaded_grammars
path = os.path.join(os.path.dirname(__file__), file) + '.txt'
try:
return _loaded_grammars[path]
except KeyError:
return _loaded_grammars.setdefault(path, generate_grammar(path))
class ErrorStatement(object):
def __init__(self, stack, next_token, position_modifier, next_start_pos):
self.stack = stack
self._position_modifier = position_modifier
self.next_token = next_token
self._next_start_pos = next_start_pos
@property
def next_start_pos(self):
s = self._next_start_pos
return s[0] + self._position_modifier.line, s[1]
@property
def first_pos(self):
first_type, nodes = self.stack[0]
return nodes[0].start_pos
@property
def first_type(self):
first_type, nodes = self.stack[0]
return first_type
class ParserSyntaxError(object):
def __init__(self, message, position):
self.message = message
self.position = position
class Parser(object):
"""
This class is used to parse a Python file, it then divides them into a
class structure of different scopes.
:param source: The codebase for the parser.
:type source: str
:param grammar: The grammar object of pgen2. Loaded by load_grammar.
:param source: The codebase for the parser. Must be unicode.
:param module_path: The path of the module in the file system, may be None.
:type module_path: str
:param no_docstr: If True, a string at the beginning is not a docstr.
:param top_module: Use this module as a parent instead of `self.module`.
"""
def __init__(self, source, module_path=None, no_docstr=False,
tokenizer=None, top_module=None):
self.no_docstr = no_docstr
def __init__(self, grammar, source, module_path=None, tokenizer=None):
self._ast_mapping = {
'expr_stmt': pt.ExprStmt,
'classdef': pt.Class,
'funcdef': pt.Function,
'file_input': pt.Module,
'import_name': pt.ImportName,
'import_from': pt.ImportFrom,
'break_stmt': pt.KeywordStatement,
'continue_stmt': pt.KeywordStatement,
'return_stmt': pt.ReturnStmt,
'raise_stmt': pt.KeywordStatement,
'yield_expr': pt.YieldExpr,
'del_stmt': pt.KeywordStatement,
'pass_stmt': pt.KeywordStatement,
'global_stmt': pt.GlobalStmt,
'nonlocal_stmt': pt.KeywordStatement,
'assert_stmt': pt.AssertStmt,
'if_stmt': pt.IfStmt,
'with_stmt': pt.WithStmt,
'for_stmt': pt.ForStmt,
'while_stmt': pt.WhileStmt,
'try_stmt': pt.TryStmt,
'comp_for': pt.CompFor,
'decorator': pt.Decorator,
'lambdef': pt.Lambda,
'old_lambdef': pt.Lambda,
'lambdef_nocond': pt.Lambda,
}
self.syntax_errors = []
self._global_names = []
self._omit_dedent_list = []
self._indent_counter = 0
self._last_failed_start_pos = (0, 0)
# TODO do print absolute import detection here.
#try:
# del python_grammar_no_print_statement.keywords["print"]
#except KeyError:
# pass # Doesn't exist in the Python 3 grammar.
#if self.options["print_function"]:
# python_grammar = pygram.python_grammar_no_print_statement
#else:
self._used_names = {}
self._scope_names_stack = [{}]
self._error_statement_stacks = []
added_newline = False
# The Python grammar needs a newline at the end of each statement.
if not source.endswith('\n'):
source += '\n'
added_newline = True
# For the fast parser.
self.position_modifier = pt.PositionModifier()
p = PgenParser(grammar, self.convert_node, self.convert_leaf,
self.error_recovery)
tokenizer = tokenizer or tokenize.source_tokens(source)
self._gen = PushBackTokenizer(tokenizer)
self.module = p.parse(self._tokenize(tokenizer))
if self.module.type != 'file_input':
# If there's only one statement, we get back a non-module. That's
# not what we want, we want a module, so we add it here:
self.module = self.convert_node(grammar,
grammar.symbol2number['file_input'],
[self.module])
# initialize global Scope
start_pos = next(self._gen).start_pos
self._gen.push_last_back()
self.module = pr.SubModule(module_path, start_pos, top_module)
self._scope = self.module
self._top_module = top_module or self.module
if added_newline:
self.remove_last_newline()
self.module.used_names = self._used_names
self.module.path = module_path
self.module.global_names = self._global_names
self.module.error_statement_stacks = self._error_statement_stacks
def convert_node(self, grammar, type, children):
"""
Convert raw node information to a Node instance.
This is passed to the parser driver which calls it whenever a reduction of a
grammar rule produces a new complete node, so that the tree is build
strictly bottom-up.
"""
symbol = grammar.number2symbol[type]
try:
self._parse()
except (common.MultiLevelStopIteration, StopIteration):
# StopIteration needs to be added as well, because python 2 has a
# strange way of handling StopIterations.
# sometimes StopIteration isn't catched. Just ignore it.
new_node = self._ast_mapping[symbol](children)
except KeyError:
new_node = pt.Node(symbol, children)
# on finish, set end_pos correctly
pass
s = self._scope
while s is not None:
s.end_pos = self._gen.current.end_pos
s = s.parent
# We need to check raw_node always, because the same node can be
# returned by convert multiple times.
if symbol == 'global_stmt':
self._global_names += new_node.get_global_names()
elif isinstance(new_node, pt.Lambda):
new_node.names_dict = self._scope_names_stack.pop()
elif isinstance(new_node, (pt.ClassOrFunc, pt.Module)) \
and symbol in ('funcdef', 'classdef', 'file_input'):
# scope_name_stack handling
scope_names = self._scope_names_stack.pop()
if isinstance(new_node, pt.ClassOrFunc):
n = new_node.name
scope_names[n.value].remove(n)
# Set the func name of the current node
arr = self._scope_names_stack[-1].setdefault(n.value, [])
arr.append(n)
new_node.names_dict = scope_names
elif isinstance(new_node, pt.CompFor):
# The name definitions of comprehenions shouldn't be part of the
# current scope. They are part of the comprehension scope.
for n in new_node.get_defined_names():
self._scope_names_stack[-1][n.value].remove(n)
return new_node
# clean up unused decorators
for d in self._decorators:
# set a parent for unused decorators, avoid NullPointerException
# because of `self.module.used_names`.
d.parent = self.module
def convert_leaf(self, grammar, type, value, prefix, start_pos):
#print('leaf', value, pytree.type_repr(type))
if type == tokenize.NAME:
if value in grammar.keywords:
if value in ('def', 'class', 'lambda'):
self._scope_names_stack.append({})
self.module.end_pos = self._gen.current.end_pos
if self._gen.current.type == tokenize.NEWLINE:
# This case is only relevant with the FastTokenizer, because
# otherwise there's always an ENDMARKER.
# we added a newline before, so we need to "remove" it again.
#
# NOTE: It should be keep end_pos as-is if the last token of
# a source is a NEWLINE, otherwise the newline at the end of
# a source is not included in a ParserNode.code.
if self._gen.previous.type != tokenize.NEWLINE:
self.module.end_pos = self._gen.previous.end_pos
return pt.Keyword(self.position_modifier, value, start_pos, prefix)
else:
name = pt.Name(self.position_modifier, value, start_pos, prefix)
# Keep a listing of all used names
arr = self._used_names.setdefault(name.value, [])
arr.append(name)
arr = self._scope_names_stack[-1].setdefault(name.value, [])
arr.append(name)
return name
elif type == STRING:
return pt.String(self.position_modifier, value, start_pos, prefix)
elif type == NUMBER:
return pt.Number(self.position_modifier, value, start_pos, prefix)
elif type in (NEWLINE, ENDMARKER):
return pt.Whitespace(self.position_modifier, value, start_pos, prefix)
else:
return pt.Operator(self.position_modifier, value, start_pos, prefix)
del self._gen
def error_recovery(self, grammar, stack, typ, value, start_pos, prefix,
add_token_callback):
"""
This parser is written in a dynamic way, meaning that this parser
allows using different grammars (even non-Python). However, error
recovery is purely written for Python.
"""
def current_suite(stack):
# For now just discard everything that is not a suite or
# file_input, if we detect an error.
for index, (dfa, state, (typ, nodes)) in reversed(list(enumerate(stack))):
# `suite` can sometimes be only simple_stmt, not stmt.
symbol = grammar.number2symbol[typ]
if symbol == 'file_input':
break
elif symbol == 'suite' and len(nodes) > 1:
# suites without an indent in them get discarded.
break
elif symbol == 'simple_stmt' and len(nodes) > 1:
# simple_stmt can just be turned into a Node, if there are
# enough statements. Ignore the rest after that.
break
return index, symbol, nodes
index, symbol, nodes = current_suite(stack)
if symbol == 'simple_stmt':
index -= 2
(_, _, (typ, suite_nodes)) = stack[index]
symbol = grammar.number2symbol[typ]
suite_nodes.append(pt.Node(symbol, list(nodes)))
# Remove
nodes[:] = []
nodes = suite_nodes
stack[index]
#print('err', token.tok_name[typ], repr(value), start_pos, len(stack), index)
self._stack_removal(grammar, stack, index + 1, value, start_pos)
if typ == INDENT:
# For every deleted INDENT we have to delete a DEDENT as well.
# Otherwise the parser will get into trouble and DEDENT too early.
self._omit_dedent_list.append(self._indent_counter)
if value in ('import', 'from', 'class', 'def', 'try', 'while', 'return'):
# Those can always be new statements.
add_token_callback(typ, value, prefix, start_pos)
elif typ == DEDENT and symbol == 'suite':
# Close the current suite, with DEDENT.
# Note that this may cause some suites to not contain any
# statements at all. This is contrary to valid Python syntax. We
# keep incomplete suites in Jedi to be able to complete param names
# or `with ... as foo` names. If we want to use this parser for
# syntax checks, we have to check in a separate turn if suites
# contain statements or not. However, a second check is necessary
# anyway (compile.c does that for Python), because Python's grammar
# doesn't stop you from defining `continue` in a module, etc.
add_token_callback(typ, value, prefix, start_pos)
def _stack_removal(self, grammar, stack, start_index, value, start_pos):
def clear_names(children):
for c in children:
try:
clear_names(c.children)
except AttributeError:
if isinstance(c, pt.Name):
try:
self._scope_names_stack[-1][c.value].remove(c)
self._used_names[c.value].remove(c)
except ValueError:
pass # This may happen with CompFor.
for dfa, state, node in stack[start_index:]:
clear_names(children=node[1])
failed_stack = []
found = False
for dfa, state, (typ, nodes) in stack[start_index:]:
if nodes:
found = True
if found:
symbol = grammar.number2symbol[typ]
failed_stack.append((symbol, nodes))
if nodes and nodes[0] in ('def', 'class', 'lambda'):
self._scope_names_stack.pop()
if failed_stack:
err = ErrorStatement(failed_stack, value, self.position_modifier, start_pos)
self._error_statement_stacks.append(err)
self._last_failed_start_pos = start_pos
stack[start_index:] = []
def _tokenize(self, tokenizer):
for typ, value, start_pos, prefix in tokenizer:
#print(tokenize.tok_name[typ], repr(value), start_pos, repr(prefix))
if typ == DEDENT:
# We need to count indents, because if we just omit any DEDENT,
# we might omit them in the wrong place.
o = self._omit_dedent_list
if o and o[-1] == self._indent_counter:
o.pop()
continue
self._indent_counter -= 1
elif typ == INDENT:
self._indent_counter += 1
elif typ == ERRORTOKEN:
self._add_syntax_error('Strange token', start_pos)
continue
if typ == OP:
typ = token.opmap[value]
yield typ, value, prefix, start_pos
def _add_syntax_error(self, message, position):
self.syntax_errors.append(ParserSyntaxError(message, position))
def __repr__(self):
return "<%s: %s>" % (type(self).__name__, self.module)
def _check_user_stmt(self, simple):
# this is not user checking, just update the used_names
for tok_name in self.module.temp_used_names:
try:
self.module.used_names[tok_name].add(simple)
except KeyError:
self.module.used_names[tok_name] = set([simple])
self.module.temp_used_names = []
def _parse_dot_name(self, pre_used_token=None):
def remove_last_newline(self):
"""
The dot name parser parses a name, variable or function and returns
their names.
:return: tuple of Name, next_token
In all of this we need to work with _start_pos, because if we worked
with start_pos, we would need to check the position_modifier as well
(which is accounted for in the start_pos property).
"""
def append(el):
names.append(el)
self.module.temp_used_names.append(el[0])
names = []
tok = next(self._gen) if pre_used_token is None else pre_used_token
if tok.type != tokenize.NAME and tok.string != '*':
return None, tok
first_pos = tok.start_pos
append((tok.string, first_pos))
while True:
end_pos = tok.end_pos
tok = next(self._gen)
if tok.string != '.':
break
tok = next(self._gen)
if tok.type != tokenize.NAME:
break
append((tok.string, tok.start_pos))
n = pr.Name(self.module, names, first_pos, end_pos) if names else None
return n, tok
def _parse_import_list(self):
"""
The parser for the imports. Unlike the class and function parse
function, this returns no Import class, but rather an import list,
which is then added later on.
The reason, why this is not done in the same class lies in the nature
of imports. There are two ways to write them:
- from ... import ...
- import ...
To distinguish, this has to be processed after the parser.
:return: List of imports.
:rtype: list
"""
imports = []
brackets = False
continue_kw = [",", ";", "\n", '\r\n', ')'] \
+ list(set(keyword.kwlist) - set(['as']))
while True:
defunct = False
tok = next(self._gen)
if tok.string == '(': # python allows only one `(` in the statement.
brackets = True
tok = next(self._gen)
if brackets and tok.type == tokenize.NEWLINE:
tok = next(self._gen)
i, tok = self._parse_dot_name(tok)
if not i:
defunct = True
name2 = None
if tok.string == 'as':
name2, tok = self._parse_dot_name()
imports.append((i, name2, defunct))
while tok.string not in continue_kw:
tok = next(self._gen)
if not (tok.string == "," or brackets and tok.type == tokenize.NEWLINE):
break
return imports
def _parse_parentheses(self, is_class):
"""
Functions and Classes have params (which means for classes
super-classes). They are parsed here and returned as Statements.
:return: List of Statements
:rtype: list
"""
params = []
tok = None
pos = 0
breaks = [',', ':']
while tok is None or tok.string not in (')', ':'):
# Classes don't have params, a Class works more like a function
# call.
param, tok = self._parse_statement(added_breaks=breaks,
stmt_class=pr.ExprStmt
if is_class else pr.Param)
if is_class:
if param is not None:
params.append(param)
else:
if param is not None and tok.string == ':':
# parse annotations
annotation, tok = self._parse_statement(added_breaks=breaks)
if annotation:
param.add_annotation(annotation)
# Function params without vars are usually syntax errors.
# expressions are valid in superclass declarations.
if param is not None and param.get_defined_names():
param.position_nr = pos
params.append(param)
pos += 1
return params
def _parse_function(self):
"""
The parser for a text functions. Process the tokens, which follow a
function definition.
:return: Return a Scope representation of the tokens.
:rtype: Function
"""
first_pos = self._gen.current.start_pos
tok = next(self._gen)
if tok.type != tokenize.NAME:
return None
fname = pr.Name(self.module, [(tok.string, tok.start_pos)], tok.start_pos,
tok.end_pos)
tok = next(self._gen)
if tok.string != '(':
return None
params = self._parse_parentheses(is_class=False)
colon = next(self._gen)
annotation = None
if colon.string in ('-', '->'):
# parse annotations
if colon.string == '-':
# The Python 2 tokenizer doesn't understand this
colon = next(self._gen)
if colon.string != '>':
return None
annotation, colon = self._parse_statement(added_breaks=[':'])
if colon.string != ':':
return None
# because of 2 line func param definitions
return pr.Function(self.module, fname, params, first_pos, annotation)
def _parse_class(self):
"""
The parser for a text class. Process the tokens, which follow a
class definition.
:return: Return a Scope representation of the tokens.
:rtype: Class
"""
first_pos = self._gen.current.start_pos
cname = next(self._gen)
if cname.type != tokenize.NAME:
debug.warning("class: syntax err, token is not a name@%s (%s: %s)",
cname.start_pos[0], tokenize.tok_name[cname.type], cname.string)
return None
cname = pr.Name(self.module, [(cname.string, cname.start_pos)],
cname.start_pos, cname.end_pos)
superclasses = []
_next = next(self._gen)
if _next.string == '(':
superclasses = self._parse_parentheses(is_class=True)
_next = next(self._gen)
if _next.string != ':':
debug.warning("class syntax: %s@%s", cname, _next.start_pos[0])
return None
return pr.Class(self.module, cname, superclasses, first_pos)
def _parse_statement(self, pre_used_token=None, added_breaks=None,
stmt_class=pr.ExprStmt, names_are_set_vars=False,
maybe_docstr=False):
"""
Parses statements like::
a = test(b)
a += 3 - 2 or b
and so on. One line at a time.
:param pre_used_token: The pre parsed token.
:type pre_used_token: set
:return: ExprStmt + last parsed token.
:rtype: (ExprStmt, str)
"""
set_vars = []
level = 0 # The level of parentheses
if pre_used_token:
tok = pre_used_token
endmarker = self.module.children[-1]
# The newline is either in the endmarker as a prefix or the previous
# leaf as a newline token.
if endmarker.prefix.endswith('\n'):
endmarker.prefix = endmarker.prefix[:-1]
last_line = re.sub('.*\n', '', endmarker.prefix)
endmarker._start_pos = endmarker._start_pos[0] - 1, len(last_line)
else:
tok = next(self._gen)
while tok.type == tokenize.COMMENT:
# remove newline and comment
next(self._gen)
tok = next(self._gen)
first_pos = tok.start_pos
opening_brackets = ['{', '(', '[']
closing_brackets = ['}', ')', ']']
# the difference between "break" and "always break" is that the latter
# will even break in parentheses. This is true for typical flow
# commands like def and class and the imports, which will never be used
# in a statement.
breaks = set(['\n', '\r\n', ':', ')'])
always_break = [';', 'import', 'from', 'class', 'def', 'try', 'except',
'finally', 'while', 'return', 'yield']
not_first_break = ['del', 'raise']
if added_breaks:
breaks |= set(added_breaks)
tok_list = []
as_names = []
in_lambda_param = False
while not (tok.string in always_break
or tok.string in not_first_break and not tok_list
or tok.string in breaks and level <= 0
and not (in_lambda_param and tok.string in ',:')):
try:
is_kw = tok.string in OPERATOR_KEYWORDS
if tok.type == tokenize.OP or is_kw:
tok_list.append(
pr.Operator(self.module, tok.string, self._scope, tok.start_pos)
)
else:
tok_list.append(tok)
if tok.string == 'as':
tok = next(self._gen)
if tok.type == tokenize.NAME:
n, tok = self._parse_dot_name(self._gen.current)
if n:
set_vars.append(n)
as_names.append(n)
tok_list.append(n)
continue
elif tok.string == 'lambda':
breaks.discard(':')
in_lambda_param = True
elif in_lambda_param and tok.string == ':':
in_lambda_param = False
elif tok.type == tokenize.NAME and not is_kw:
n, tok = self._parse_dot_name(self._gen.current)
# removed last entry, because we add Name
tok_list.pop()
if n:
tok_list.append(n)
continue
elif tok.string in opening_brackets:
level += 1
elif tok.string in closing_brackets:
level -= 1
tok = next(self._gen)
except (StopIteration, common.MultiLevelStopIteration):
# comes from tokenizer
break
if not tok_list:
return None, tok
first_tok = tok_list[0]
# docstrings
if len(tok_list) == 1 and isinstance(first_tok, tokenize.Token) \
and first_tok.type == tokenize.STRING and maybe_docstr:
# Normal docstring check
if self.freshscope and not self.no_docstr:
self._scope.add_docstr(first_tok)
return None, tok
# Attribute docstring (PEP 224) support (sphinx uses it, e.g.)
# If string literal is being parsed...
else:
with common.ignored(IndexError, AttributeError):
# ...then set it as a docstring
self._scope.statements[-1].add_docstr(first_tok)
return None, tok
stmt = stmt_class(self.module, tok_list, first_pos, tok.end_pos,
as_names=as_names,
names_are_set_vars=names_are_set_vars)
stmt.parent = self._top_module
self._check_user_stmt(stmt)
if tok.string in always_break + not_first_break:
self._gen.push_last_back()
return stmt, tok
def _parse(self):
"""
The main part of the program. It analyzes the given code-text and
returns a tree-like scope. For a more detailed description, see the
class description.
:param text: The code which should be parsed.
:param type: str
:raises: IndentationError
"""
extended_flow = ['else', 'elif', 'except', 'finally']
statement_toks = ['{', '[', '(', '`']
self._decorators = []
self.freshscope = True
for tok in self._gen:
token_type = tok.type
tok_str = tok.string
first_pos = tok.start_pos
self.module.temp_used_names = []
# debug.dbg('main: tok=[%s] type=[%s] indent=[%s]', \
# tok, tokenize.tok_name[token_type], start_position[0])
# check again for unindented stuff. this is true for syntax
# errors. only check for names, because thats relevant here. If
# some docstrings are not indented, I don't care.
while first_pos[1] <= self._scope.start_pos[1] \
and (token_type == tokenize.NAME or tok_str in ('(', '['))\
and self._scope != self.module:
self._scope.end_pos = first_pos
self._scope = self._scope.parent
if isinstance(self._scope, pr.Module) \
and not isinstance(self._scope, pr.SubModule):
self._scope = self.module
if isinstance(self._scope, pr.SubModule):
use_as_parent_scope = self._top_module
else:
use_as_parent_scope = self._scope
if tok_str == 'def':
func = self._parse_function()
if func is None:
debug.warning("function: syntax error@%s", first_pos[0])
continue
self.freshscope = True
self._scope = self._scope.add_scope(func, self._decorators)
self._decorators = []
elif tok_str == 'class':
cls = self._parse_class()
if cls is None:
debug.warning("class: syntax error@%s" % first_pos[0])
continue
self.freshscope = True
self._scope = self._scope.add_scope(cls, self._decorators)
self._decorators = []
# import stuff
elif tok_str == 'import':
imports = self._parse_import_list()
for count, (m, alias, defunct) in enumerate(imports):
e = (alias or m or self._gen.previous).end_pos
end_pos = self._gen.previous.end_pos if count + 1 == len(imports) else e
i = pr.Import(self.module, first_pos, end_pos, m,
alias, defunct=defunct)
self._check_user_stmt(i)
self._scope.add_import(i)
if not imports:
i = pr.Import(self.module, first_pos, self._gen.current.end_pos,
None, defunct=True)
self._check_user_stmt(i)
self.freshscope = False
elif tok_str == 'from':
defunct = False
# take care for relative imports
relative_count = 0
while True:
tok = next(self._gen)
if tok.string != '.':
break
relative_count += 1
# the from import
mod, tok = self._parse_dot_name(self._gen.current)
tok_str = tok.string
if str(mod) == 'import' and relative_count:
self._gen.push_last_back()
tok_str = 'import'
mod = None
if not mod and not relative_count or tok_str != "import":
debug.warning("from: syntax error@%s", tok.start_pos[0])
defunct = True
if tok_str != 'import':
self._gen.push_last_back()
names = self._parse_import_list()
for count, (name, alias, defunct2) in enumerate(names):
star = name is not None and unicode(name.names[0]) == '*'
if star:
name = None
e = (alias or name or self._gen.previous).end_pos
end_pos = self._gen.previous.end_pos if count + 1 == len(names) else e
i = pr.Import(self.module, first_pos, end_pos, name,
alias, mod, star, relative_count,
defunct=defunct or defunct2)
self._check_user_stmt(i)
self._scope.add_import(i)
self.freshscope = False
# loops
elif tok_str == 'for':
set_stmt, tok = self._parse_statement(added_breaks=['in'],
names_are_set_vars=True)
if tok.string != 'in':
debug.warning('syntax err, for flow incomplete @%s', tok.start_pos[0])
try:
statement, tok = self._parse_statement()
except StopIteration:
statement, tok = None, None
s = [] if statement is None else [statement]
f = pr.ForFlow(self.module, s, first_pos, set_stmt)
self._scope = self._scope.add_statement(f)
if tok is None or tok.string != ':':
debug.warning('syntax err, for flow started @%s', first_pos[0])
elif tok_str in ['if', 'while', 'try', 'with'] + extended_flow:
added_breaks = []
command = tok_str
if command in ('except', 'with'):
added_breaks.append(',')
# multiple inputs because of with
inputs = []
first = True
while first or command == 'with' and tok.string not in (':', '\n', '\r\n'):
statement, tok = \
self._parse_statement(added_breaks=added_breaks)
if command == 'except' and tok.string == ',':
# the except statement defines a var
# this is only true for python 2
n, tok = self._parse_dot_name()
if n:
n.parent = statement
statement.as_names.append(n)
if statement:
inputs.append(statement)
first = False
f = pr.Flow(self.module, command, inputs, first_pos)
if command in extended_flow:
# The last statement has to be another part of the flow
# statement, because a dedent releases the main scope, so
# just take the last statement.
newline = endmarker.get_previous()
except IndexError:
return # This means that the parser is empty.
while True:
if newline.value == '':
# Must be a DEDENT, just continue.
try:
s = self._scope.statements[-1].set_next(f)
except (AttributeError, IndexError):
# If set_next doesn't exist, just add it.
s = self._scope.add_statement(f)
newline = newline.get_previous()
except IndexError:
# If there's a statement that fails to be parsed, there
# will be no previous leaf. So just ignore it.
break
elif newline.value != '\n':
# This may happen if error correction strikes and removes
# a whole statement including '\n'.
break
else:
s = self._scope.add_statement(f)
self._scope = s
if tok.string != ':':
debug.warning('syntax err, flow started @%s', tok.start_pos[0])
# returns
elif tok_str in ('return', 'yield'):
s = tok.start_pos
self.freshscope = False
# Add returns to the scope
# Should be a function, otherwise just add it to a module!
func = self._scope.get_parent_until((pr.Function, pr.Module))
if tok_str == 'yield':
func.is_generator = True
stmt, tok = self._parse_statement()
if stmt is not None:
stmt.parent = use_as_parent_scope
try:
kw_stmt = pr.KeywordStatement(tok_str, s,
use_as_parent_scope, stmt)
self._scope.statements.append(kw_stmt)
func.returns.append(kw_stmt)
# start_pos is the one of the return statement
stmt.start_pos = s
except AttributeError:
debug.warning('return in non-function')
stmt = None
elif tok_str == 'assert':
stmt, tok = self._parse_statement()
if stmt is not None:
stmt.parent = use_as_parent_scope
self._scope.statements.append(stmt)
self._scope.asserts.append(stmt)
elif tok_str in STATEMENT_KEYWORDS:
stmt, _ = self._parse_statement()
kw = pr.KeywordStatement(tok_str, tok.start_pos,
use_as_parent_scope, stmt)
self._scope.add_statement(kw)
if stmt is not None and tok_str == 'global':
for t in stmt._token_list:
if isinstance(t, pr.Name):
# Add the global to the top module, it counts there.
self.module.add_global(t)
# decorator
elif tok_str == '@':
stmt, tok = self._parse_statement()
if stmt is not None:
self._decorators.append(stmt)
elif tok_str == 'pass':
continue
# default
elif token_type in (tokenize.NAME, tokenize.STRING,
tokenize.NUMBER, tokenize.OP) \
or tok_str in statement_toks:
# this is the main part - a name can be a function or a
# normal var, which can follow anything. but this is done
# by the statement parser.
stmt, tok = self._parse_statement(self._gen.current,
maybe_docstr=True)
if stmt:
self._scope.add_statement(stmt)
self.freshscope = False
else:
if token_type not in (tokenize.COMMENT, tokenize.NEWLINE, tokenize.ENDMARKER):
debug.warning('Token not used: %s %s %s', tok_str,
tokenize.tok_name[token_type], first_pos)
continue
self.no_docstr = False
class PushBackTokenizer(object):
def __init__(self, tokenizer):
self._tokenizer = tokenizer
self._push_backs = []
self.current = self.previous = tokenize.Token(None, '', (0, 0))
def push_last_back(self):
self._push_backs.append(self.current)
def next(self):
""" Python 2 Compatibility """
return self.__next__()
def __next__(self):
if self._push_backs:
return self._push_backs.pop(0)
previous = self.current
self.current = next(self._tokenizer)
self.previous = previous
return self.current
def __iter__(self):
return self
newline.value = ''
if self._last_failed_start_pos > newline._start_pos:
# It may be the case that there was a syntax error in a
# function. In that case error correction removes the
# right newline. So we use the previously assigned
# _last_failed_start_pos variable to account for that.
endmarker._start_pos = self._last_failed_start_pos
else:
endmarker._start_pos = newline._start_pos
break
jedi/parser/fast.py
+434 -320
View File
@@ -4,65 +4,106 @@ anything changes, it only reparses the changed parts. But because it's not
finished (and still not working as I want), I won't document it any further.
"""
import re
from itertools import chain
from jedi._compatibility import use_metaclass, unicode
from jedi._compatibility import use_metaclass
from jedi import settings
from jedi import common
from jedi.parser import Parser
from jedi.parser import representation as pr
from jedi.parser import tokenize
from jedi.parser import tree as pr
from jedi import cache
from jedi.parser.tokenize import (source_tokens, Token, FLOWS, NEWLINE,
COMMENT, ENDMARKER)
from jedi import debug
from jedi.parser.tokenize import (source_tokens, NEWLINE,
ENDMARKER, INDENT, DEDENT)
FLOWS = 'if', 'else', 'elif', 'while', 'with', 'try', 'except', 'finally', 'for'
class Module(pr.Module, pr.Simple):
def __init__(self, parsers):
super(Module, self).__init__(self, (1, 0))
self.parsers = parsers
class FastModule(pr.Module):
type = 'file_input'
def __init__(self, module_path):
super(FastModule, self).__init__([])
self.modules = []
self.reset_caches()
self.start_pos = 1, 0
self.end_pos = None, None
self.names_dict = {}
self.path = module_path
def reset_caches(self):
""" This module does a whole lot of caching, because it uses different
parsers. """
with common.ignored(AttributeError):
del self._used_names
def __getattr__(self, name):
if name.startswith('__'):
raise AttributeError('Not available!')
else:
return getattr(self.parsers[0].module, name)
self.modules = []
try:
del self._used_names # Remove the used names cache.
except AttributeError:
pass # It was never used.
@property
@cache.underscore_memoization
def used_names(self):
used_names = {}
for p in self.parsers:
for k, statement_set in p.module.used_names.items():
if k in used_names:
used_names[k] |= statement_set
else:
used_names[k] = set(statement_set)
return used_names
return MergedNamesDict([m.used_names for m in self.modules])
@property
def global_names(self):
return [name for m in self.modules for name in m.global_names]
@property
def error_statement_stacks(self):
return [e for m in self.modules for e in m.error_statement_stacks]
def __repr__(self):
return "<fast.%s: %s@%s-%s>" % (type(self).__name__, self.name,
self.start_pos[0], self.end_pos[0])
# To avoid issues with with the `parser.Parser`, we need setters that do
# nothing, because if pickle comes along and sets those values.
@global_names.setter
def global_names(self, value):
pass
@error_statement_stacks.setter
def error_statement_stacks(self, value):
pass
@used_names.setter
def used_names(self, value):
pass
class MergedNamesDict(object):
def __init__(self, dicts):
self.dicts = dicts
def __iter__(self):
return iter(set(key for dct in self.dicts for key in dct))
def __getitem__(self, value):
return list(chain.from_iterable(dct.get(value, []) for dct in self.dicts))
def items(self):
dct = {}
for d in self.dicts:
for key, values in d.items():
try:
dct_values = dct[key]
dct_values += values
except KeyError:
dct[key] = list(values)
return dct.items()
def values(self):
lst = []
for dct in self.dicts:
lst += dct.values()
return lst
class CachedFastParser(type):
""" This is a metaclass for caching `FastParser`. """
def __call__(self, source, module_path=None):
def __call__(self, grammar, source, module_path=None):
if not settings.fast_parser:
return Parser(source, module_path)
return Parser(grammar, source, module_path)
pi = cache.parser_cache.get(module_path, None)
if pi is None or isinstance(pi.parser, Parser):
p = super(CachedFastParser, self).__call__(source, module_path)
p = super(CachedFastParser, self).__call__(grammar, source, module_path)
else:
p = pi.parser # pi is a `cache.ParserCacheItem`
p.update(source)
@@ -70,397 +111,470 @@ class CachedFastParser(type):
class ParserNode(object):
def __init__(self, parser, code, parent=None):
self.parent = parent
def __init__(self, fast_module, parser, source):
self._fast_module = fast_module
self.parent = None
self._node_children = []
self.children = []
# must be created before new things are added to it.
self.save_contents(parser, code)
def save_contents(self, parser, code):
self.code = code
self.hash = hash(code)
self.source = source
self.hash = hash(source)
self.parser = parser
try:
# with fast_parser we have either 1 subscope or only statements.
self.content_scope = parser.module.subscopes[0]
# With fast_parser we have either 1 subscope or only statements.
self._content_scope = parser.module.subscopes[0]
except IndexError:
self.content_scope = parser.module
self._content_scope = parser.module
else:
self._rewrite_last_newline()
scope = self.content_scope
self._contents = {}
for c in pr.SCOPE_CONTENTS:
self._contents[c] = list(getattr(scope, c))
self._is_generator = scope.is_generator
# We need to be able to reset the original children of a parser.
self._old_children = list(self._content_scope.children)
self.old_children = self.children
self.children = []
def _rewrite_last_newline(self):
"""
The ENDMARKER can contain a newline in the prefix. However this prefix
really belongs to the function - respectively to the next function or
parser node. If we don't rewrite that newline, we end up with a newline
in the wrong position, i.d. at the end of the file instead of in the
middle.
"""
c = self._content_scope.children
if pr.is_node(c[-1], 'suite'): # In a simple_stmt there's no DEDENT.
end_marker = self.parser.module.children[-1]
# Set the DEDENT prefix instead of the ENDMARKER.
c[-1].children[-1].prefix = end_marker.prefix
end_marker.prefix = ''
def reset_contents(self):
scope = self.content_scope
for key, c in self._contents.items():
setattr(scope, key, list(c))
scope.is_generator = self._is_generator
def __repr__(self):
module = self.parser.module
try:
return '<%s: %s-%s>' % (type(self).__name__, module.start_pos, module.end_pos)
except IndexError:
# There's no module yet.
return '<%s: empty>' % type(self).__name__
if self.parent is None:
# Global vars of the first one can be deleted, in the global scope
# they make no sense.
self.parser.module.global_vars = []
def reset_node(self):
"""
Removes changes that were applied in this class.
"""
self._node_children = []
scope = self._content_scope
scope.children = list(self._old_children)
try:
# This works if it's a MergedNamesDict.
# We are correcting it, because the MergedNamesDicts are artificial
# and can change after closing a node.
scope.names_dict = scope.names_dict.dicts[0]
except AttributeError:
pass
for c in self.children:
c.reset_contents()
def close(self):
"""
Closes the current parser node. This means that after this no further
nodes should be added anymore.
"""
# We only need to replace the dict if multiple dictionaries are used:
if self._node_children:
dcts = [n.parser.module.names_dict for n in self._node_children]
# Need to insert the own node as well.
dcts.insert(0, self._content_scope.names_dict)
self._content_scope.names_dict = MergedNamesDict(dcts)
def parent_until_indent(self, indent=None):
if indent is None or self.indent >= indent and self.parent:
self.old_children = []
if self.parent is not None:
return self.parent.parent_until_indent(indent)
if (indent is None or self._indent >= indent) and self.parent is not None:
self.close()
return self.parent.parent_until_indent(indent)
return self
@property
def indent(self):
def _indent(self):
if not self.parent:
return 0
module = self.parser.module
try:
el = module.subscopes[0]
except IndexError:
try:
el = module.statements[0]
except IndexError:
try:
el = module.imports[0]
except IndexError:
try:
el = [r for r in module.returns if r is not None][0]
except IndexError:
return self.parent.indent + 1
return el.start_pos[1]
def _set_items(self, parser, set_parent=False):
# insert parser objects into current structure
scope = self.content_scope
for c in pr.SCOPE_CONTENTS:
content = getattr(scope, c)
items = getattr(parser.module, c)
if set_parent:
for i in items:
if i is None:
continue # happens with empty returns
i.parent = scope.use_as_parent
if isinstance(i, (pr.Function, pr.Class)):
for d in i.decorators:
d.parent = scope.use_as_parent
content += items
return self.parser.module.children[0].start_pos[1]
# global_vars
cur = self
while cur.parent is not None:
cur = cur.parent
cur.parser.module.global_vars += parser.module.global_vars
scope.is_generator |= parser.module.is_generator
def add_node(self, node, set_parent=False):
def add_node(self, node, line_offset):
"""Adding a node means adding a node that was already added earlier"""
self.children.append(node)
self._set_items(node.parser, set_parent=set_parent)
node.old_children = node.children # TODO potential memory leak?
node.children = []
# Changing the line offsets is very important, because if they don't
# fit, all the start_pos values will be wrong.
m = node.parser.module
node.parser.position_modifier.line = line_offset
self._fast_module.modules.append(m)
node.parent = self
self._node_children.append(node)
# Insert parser objects into current structure. We only need to set the
# parents and children in a good way.
scope = self._content_scope
for child in m.children:
child.parent = scope
scope.children.append(child)
scope = self.content_scope
while scope is not None:
#print('x',scope)
if not isinstance(scope, pr.SubModule):
# TODO This seems like a strange thing. Check again.
scope.end_pos = node.content_scope.end_pos
scope = scope.parent
return node
def add_parser(self, parser, code):
return self.add_node(ParserNode(parser, code, self), True)
def all_sub_nodes(self):
"""
Returns all nodes including nested ones.
"""
for n in self._node_children:
yield n
for y in n.all_sub_nodes():
yield y
@cache.underscore_memoization # Should only happen once!
def remove_last_newline(self):
self.parser.remove_last_newline()
class FastParser(use_metaclass(CachedFastParser)):
_FLOWS_NEED_SPACE = 'if', 'elif', 'while', 'with', 'except', 'for'
_FLOWS_NEED_COLON = 'else', 'try', 'except', 'finally'
_keyword_re = re.compile('^[ \t]*(def |class |@|(?:%s)|(?:%s)\s*:)'
% ('|'.join(_FLOWS_NEED_SPACE),
'|'.join(_FLOWS_NEED_COLON)))
_keyword_re = re.compile('^[ \t]*(def|class|@|%s)' % '|'.join(tokenize.FLOWS))
def __init__(self, code, module_path=None):
def __init__(self, grammar, source, module_path=None):
# set values like `pr.Module`.
self._grammar = grammar
self.module_path = module_path
self._reset_caches()
self.update(source)
self.current_node = None
self.parsers = []
self.module = Module(self.parsers)
self.reset_caches()
def _reset_caches(self):
self.module = FastModule(self.module_path)
self.current_node = ParserNode(self.module, self, '')
def update(self, source):
# For testing purposes: It is important that the number of parsers used
# can be minimized. With these variables we can test against that.
self.number_parsers_used = 0
self.number_of_splits = 0
self.number_of_misses = 0
self.module.reset_caches()
try:
self._parse(code)
self._parse(source)
except:
# FastParser is cached, be careful with exceptions
del self.parsers[:]
# FastParser is cached, be careful with exceptions.
self._reset_caches()
raise
def update(self, code):
self.reset_caches()
try:
self._parse(code)
except:
# FastParser is cached, be careful with exceptions
del self.parsers[:]
raise
def _split_parts(self, code):
def _split_parts(self, source):
"""
Split the code into different parts. This makes it possible to parse
each part seperately and therefore cache parts of the file and not
everything.
Split the source code into different parts. This makes it possible to
parse each part seperately and therefore cache parts of the file and
not everything.
"""
def gen_part():
text = '\n'.join(current_lines)
text = ''.join(current_lines)
del current_lines[:]
self.number_of_splits += 1
return text
def just_newlines(current_lines):
for line in current_lines:
line = line.lstrip('\t \n\r')
if line and line[0] != '#':
return False
return True
# Split only new lines. Distinction between \r\n is the tokenizer's
# job.
self._lines = code.split('\n')
# It seems like there's no problem with form feed characters here,
# because we're not counting lines.
self._lines = source.splitlines(True)
current_lines = []
is_decorator = False
current_indent = 0
old_indent = 0
# Use -1, because that indent is always smaller than any other.
indent_list = [-1, 0]
new_indent = False
in_flow = False
parentheses_level = 0
flow_indent = None
previous_line = None
# All things within flows are simply being ignored.
for l in self._lines:
for i, l in enumerate(self._lines):
# Handle backslash newline escaping.
if l.endswith('\\\n') or l.endswith('\\\r\n'):
if previous_line is not None:
previous_line += l
else:
previous_line = l
continue
if previous_line is not None:
l = previous_line + l
previous_line = None
# check for dedents
s = l.lstrip('\t ')
s = l.lstrip('\t \n\r')
indent = len(l) - len(s)
if not s or s[0] in ('#', '\r'):
current_lines.append(l) # just ignore comments and blank lines
if not s or s[0] == '#':
current_lines.append(l) # Just ignore comments and blank lines
continue
if indent < current_indent: # -> dedent
current_indent = indent
new_indent = False
if not in_flow or indent < old_indent:
if current_lines:
yield gen_part()
in_flow = False
elif new_indent:
current_indent = indent
if new_indent:
if indent > indent_list[-2]:
# Set the actual indent, not just the random old indent + 1.
indent_list[-1] = indent
new_indent = False
while indent <= indent_list[-2]: # -> dedent
indent_list.pop()
# This automatically resets the flow_indent if there was a
# dedent or a flow just on one line (with one simple_stmt).
new_indent = False
if flow_indent is None and current_lines and not parentheses_level:
yield gen_part()
flow_indent = None
# Check lines for functions/classes and split the code there.
if not in_flow:
if flow_indent is None:
m = self._keyword_re.match(l)
if m:
in_flow = m.group(1) in tokenize.FLOWS
if not is_decorator and not in_flow:
if current_lines:
# Strip whitespace and colon from flows as a check.
if m.group(1).strip(' \t\r\n:') in FLOWS:
if not parentheses_level:
flow_indent = indent
else:
if not is_decorator and not just_newlines(current_lines):
yield gen_part()
is_decorator = '@' == m.group(1)
if not is_decorator:
old_indent = current_indent
current_indent += 1 # it must be higher
parentheses_level = 0
# The new indent needs to be higher
indent_list.append(indent + 1)
new_indent = True
elif is_decorator:
is_decorator = False
parentheses_level = \
max(0, (l.count('(') + l.count('[') + l.count('{')
- l.count(')') - l.count(']') - l.count('}')))
current_lines.append(l)
if current_lines:
yield gen_part()
def _parse(self, code):
""" :type code: str """
def empty_parser():
new, temp = self._get_parser(unicode(''), unicode(''), 0, [], False)
return new
def _parse(self, source):
""" :type source: str """
added_newline = False
if not source or source[-1] != '\n':
# To be compatible with Pythons grammar, we need a newline at the
# end. The parser would handle it, but since the fast parser abuses
# the normal parser in various ways, we need to care for this
# ourselves.
source += '\n'
added_newline = True
del self.parsers[:]
line_offset = 0
next_line_offset = line_offset = 0
start = 0
p = None
is_first = True
for code_part in self._split_parts(code):
if is_first or line_offset >= p.module.end_pos[0]:
indent = len(code_part) - len(code_part.lstrip('\t '))
if is_first and self.current_node is not None:
nodes = [self.current_node]
else:
nodes = []
if self.current_node is not None:
self.current_node = \
self.current_node.parent_until_indent(indent)
nodes += self.current_node.old_children
nodes = list(self.current_node.all_sub_nodes())
# Now we can reset the node, because we have all the old nodes.
self.current_node.reset_node()
last_end_line = 1
# check if code_part has already been parsed
# print '#'*45,line_offset, p and p.module.end_pos, '\n', code_part
p, node = self._get_parser(code_part, code[start:],
line_offset, nodes, not is_first)
for code_part in self._split_parts(source):
next_line_offset += code_part.count('\n')
# If the last code part parsed isn't equal to the current end_pos,
# we know that the parser went further (`def` start in a
# docstring). So just parse the next part.
if line_offset + 1 == last_end_line:
self.current_node = self._get_node(code_part, source[start:],
line_offset, nodes)
else:
# Means that some lines where not fully parsed. Parse it now.
# This is a very rare case. Should only happens with very
# strange code bits.
self.number_of_misses += 1
while last_end_line < next_line_offset + 1:
line_offset = last_end_line - 1
# We could calculate the src in a more complicated way to
# make caching here possible as well. However, this is
# complicated and error-prone. Since this is not very often
# called - just ignore it.
src = ''.join(self._lines[line_offset:])
self.current_node = self._get_node(code_part, src,
line_offset, nodes)
last_end_line = self.current_node.parser.module.end_pos[0]
# The actual used code_part is different from the given code
# part, because of docstrings for example there's a chance that
# splits are wrong.
used_lines = self._lines[line_offset:p.module.end_pos[0]]
code_part_actually_used = '\n'.join(used_lines)
debug.dbg('While parsing %s, line %s slowed down the fast parser.',
self.module_path, line_offset + 1)
if is_first and p.module.subscopes:
# special case, we cannot use a function subscope as a
# base scope, subscopes would save all the other contents
new = empty_parser()
if self.current_node is None:
self.current_node = ParserNode(new, '')
else:
self.current_node.save_contents(new, '')
self.parsers.append(new)
is_first = False
line_offset = next_line_offset
start += len(code_part)
if is_first:
if self.current_node is None:
self.current_node = ParserNode(p, code_part_actually_used)
else:
self.current_node.save_contents(p, code_part_actually_used)
else:
if node is None:
self.current_node = \
self.current_node.add_parser(p, code_part_actually_used)
else:
self.current_node = self.current_node.add_node(node)
last_end_line = self.current_node.parser.module.end_pos[0]
self.parsers.append(p)
if added_newline:
self.current_node.remove_last_newline()
is_first = False
#else:
#print '#'*45, line_offset, p.module.end_pos, 'theheck\n', repr(code_part)
# Now that the for loop is finished, we still want to close all nodes.
self.current_node = self.current_node.parent_until_indent()
self.current_node.close()
line_offset += code_part.count('\n') + 1
start += len(code_part) + 1 # +1 for newline
debug.dbg('Parsed %s, with %s parsers in %s splits.'
% (self.module_path, self.number_parsers_used,
self.number_of_splits))
if self.parsers:
self.current_node = self.current_node.parent_until_indent()
else:
self.parsers.append(empty_parser())
def _get_node(self, source, parser_code, line_offset, nodes):
"""
Side effect: Alters the list of nodes.
"""
indent = len(source) - len(source.lstrip('\t '))
self.current_node = self.current_node.parent_until_indent(indent)
self.module.end_pos = self.parsers[-1].module.end_pos
# print(self.parsers[0].module.get_code())
def _get_parser(self, code, parser_code, line_offset, nodes, no_docstr):
h = hash(code)
h = hash(source)
for index, node in enumerate(nodes):
if node.hash != h or node.code != code:
continue
if node != self.current_node:
offset = int(nodes[0] == self.current_node)
self.current_node.old_children.pop(index - offset)
p = node.parser
m = p.module
m.line_offset += line_offset + 1 - m.start_pos[0]
break
if node.hash == h and node.source == source:
node.reset_node()
nodes.remove(node)
break
else:
tokenizer = FastTokenizer(parser_code, line_offset)
p = Parser(parser_code, self.module_path, tokenizer=tokenizer,
top_module=self.module, no_docstr=no_docstr)
p.module.parent = self.module
node = None
tokenizer = FastTokenizer(parser_code)
self.number_parsers_used += 1
p = Parser(self._grammar, parser_code, self.module_path, tokenizer=tokenizer)
return p, node
end = line_offset + p.module.end_pos[0]
used_lines = self._lines[line_offset:end - 1]
code_part_actually_used = ''.join(used_lines)
def reset_caches(self):
self.module.reset_caches()
if self.current_node is not None:
self.current_node.reset_contents()
node = ParserNode(self.module, p, code_part_actually_used)
self.current_node.add_node(node, line_offset)
return node
class FastTokenizer(object):
"""
Breaks when certain conditions are met, i.e. a new function or class opens.
"""
def __init__(self, source, line_offset=0):
def __init__(self, source):
self.source = source
self.gen = source_tokens(source, line_offset)
self.closed = False
self._gen = source_tokens(source)
self._closed = False
# fast parser options
self.current = self.previous = Token(None, '', (0, 0))
self.in_flow = False
self.new_indent = False
self.parser_indent = self.old_parser_indent = 0
self.is_decorator = False
self.first_stmt = True
self.parentheses_level = 0
self.current = self.previous = NEWLINE, '', (0, 0)
self._in_flow = False
self._is_decorator = False
self._first_stmt = True
self._parentheses_level = 0
self._indent_counter = 0
self._flow_indent_counter = 0
self._returned_endmarker = False
self._expect_indent = False
def __iter__(self):
return self
def next(self):
""" Python 2 Compatibility """
return self.__next__()
def __next__(self):
if self.closed:
raise common.MultiLevelStopIteration()
if self._closed:
return self._finish_dedents()
current = next(self.gen)
tok_type = current.type
tok_str = current.string
if tok_type == ENDMARKER:
raise common.MultiLevelStopIteration()
typ, value, start_pos, prefix = current = next(self._gen)
if typ == ENDMARKER:
self._closed = True
self._returned_endmarker = True
return current
self.previous = self.current
self.current = current
# this is exactly the same check as in fast_parser, but this time with
# tokenize and therefore precise.
breaks = ['def', 'class', '@']
if typ == INDENT:
self._indent_counter += 1
if not self._expect_indent and not self._first_stmt and not self._in_flow:
# This does not mean that there is an actual flow, it means
# that the INDENT is syntactically wrong.
self._flow_indent_counter = self._indent_counter - 1
self._in_flow = True
self._expect_indent = False
elif typ == DEDENT:
self._indent_counter -= 1
if self._in_flow:
if self._indent_counter == self._flow_indent_counter:
self._in_flow = False
else:
self._closed = True
return current
def close():
if not self.first_stmt:
self.closed = True
raise common.MultiLevelStopIteration()
if value in ('def', 'class') and self._parentheses_level \
and re.search(r'\n[ \t]*\Z', prefix):
# Account for the fact that an open parentheses before a function
# will reset the parentheses counter, but new lines before will
# still be ignored. So check the prefix.
# Ignore comments/newlines, irrelevant for indentation.
if self.previous.type in (None, NEWLINE) \
and tok_type not in (COMMENT, NEWLINE):
# print c, tok_name[c[0]]
indent = current.start_pos[1]
if self.parentheses_level:
# parentheses ignore the indentation rules.
pass
elif indent < self.parser_indent: # -> dedent
self.parser_indent = indent
self.new_indent = False
if not self.in_flow or indent < self.old_parser_indent:
close()
# TODO what about flow parentheses counter resets in the tokenizer?
self._parentheses_level = 0
return self._close()
self.in_flow = False
elif self.new_indent:
self.parser_indent = indent
self.new_indent = False
# Parentheses ignore the indentation rules. The other three stand for
# new lines.
if self.previous[0] in (NEWLINE, INDENT, DEDENT) \
and not self._parentheses_level and typ not in (INDENT, DEDENT):
if not self._in_flow:
if value in FLOWS:
self._flow_indent_counter = self._indent_counter
self._first_stmt = False
elif value in ('def', 'class', '@'):
# The values here are exactly the same check as in
# _split_parts, but this time with tokenize and therefore
# precise.
if not self._first_stmt and not self._is_decorator:
return self._close()
if not self.in_flow:
if tok_str in FLOWS or tok_str in breaks:
self.in_flow = tok_str in FLOWS
if not self.is_decorator and not self.in_flow:
close()
self._is_decorator = '@' == value
if not self._is_decorator:
self._first_stmt = False
self._expect_indent = True
elif self._expect_indent:
return self._close()
else:
self._first_stmt = False
self.is_decorator = '@' == tok_str
if not self.is_decorator:
self.old_parser_indent = self.parser_indent
self.parser_indent += 1 # new scope: must be higher
self.new_indent = True
if tok_str != '@':
if self.first_stmt and not self.new_indent:
self.parser_indent = indent
self.first_stmt = False
# Ignore closing parentheses, because they are all
# irrelevant for the indentation.
if tok_str in '([{':
self.parentheses_level += 1
elif tok_str in ')]}':
self.parentheses_level = max(self.parentheses_level - 1, 0)
if value in '([{' and value:
self._parentheses_level += 1
elif value in ')]}' and value:
# Ignore closing parentheses, because they are all
# irrelevant for the indentation.
self._parentheses_level = max(self._parentheses_level - 1, 0)
return current
def _close(self):
if self._first_stmt:
# Continue like nothing has happened, because we want to enter
# the first class/function.
if self.current[1] != '@':
self._first_stmt = False
return self.current
else:
self._closed = True
return self._finish_dedents()
def _finish_dedents(self):
if self._indent_counter:
self._indent_counter -= 1
return DEDENT, '', self.current[2], ''
elif not self._returned_endmarker:
self._returned_endmarker = True
return ENDMARKER, '', self.current[2], self._get_prefix()
else:
raise StopIteration
def _get_prefix(self):
"""
We're using the current prefix for the endmarker to not loose any
information. However we care about "lost" lines. The prefix of the
current line (indent) will always be included in the current line.
"""
cur = self.current
while cur[0] == DEDENT:
cur = next(self._gen)
prefix = cur[3]
# \Z for the end of the string. $ is bugged, because it has the
# same behavior with or without re.MULTILINE.
return re.sub(r'[^\n]+\Z', '', prefix)
jedi/parser/grammar2.7.txt
+163
View File
@@ -0,0 +1,163 @@
# Grammar for 2to3. This grammar supports Python 2.x and 3.x.
# Note: Changing the grammar specified in this file will most likely
# require corresponding changes in the parser module
# (../Modules/parsermodule.c). If you can't make the changes to
# that module yourself, please co-ordinate the required changes
# with someone who can; ask around on python-dev for help. Fred
# Drake <fdrake@acm.org> will probably be listening there.
# NOTE WELL: You should also follow all the steps listed in PEP 306,
# "How to Change Python's Grammar"
# Commands for Kees Blom's railroad program
#diagram:token NAME
#diagram:token NUMBER
#diagram:token STRING
#diagram:token NEWLINE
#diagram:token ENDMARKER
#diagram:token INDENT
#diagram:output\input python.bla
#diagram:token DEDENT
#diagram:output\textwidth 20.04cm\oddsidemargin 0.0cm\evensidemargin 0.0cm
#diagram:rules
# Start symbols for the grammar:
# file_input is a module or sequence of commands read from an input file;
# single_input is a single interactive statement;
# eval_input is the input for the eval() and input() functions.
# NB: compound_stmt in single_input is followed by extra NEWLINE!
file_input: (NEWLINE | stmt)* ENDMARKER
single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE
eval_input: testlist NEWLINE* ENDMARKER
decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE
decorators: decorator+
decorated: decorators (classdef | funcdef)
funcdef: 'def' NAME parameters ['->' test] ':' suite
parameters: '(' [typedargslist] ')'
typedargslist: ((tfpdef ['=' test] ',')*
('*' [tname] (',' tname ['=' test])* [',' '**' tname] | '**' tname)
| tfpdef ['=' test] (',' tfpdef ['=' test])* [','])
tname: NAME [':' test]
tfpdef: tname | '(' tfplist ')'
tfplist: tfpdef (',' tfpdef)* [',']
varargslist: ((vfpdef ['=' test] ',')*
('*' [vname] (',' vname ['=' test])* [',' '**' vname] | '**' vname)
| vfpdef ['=' test] (',' vfpdef ['=' test])* [','])
vname: NAME
vfpdef: vname | '(' vfplist ')'
vfplist: vfpdef (',' vfpdef)* [',']
stmt: simple_stmt | compound_stmt
simple_stmt: small_stmt (';' small_stmt)* [';'] NEWLINE
small_stmt: (expr_stmt | print_stmt | del_stmt | pass_stmt | flow_stmt |
import_stmt | global_stmt | exec_stmt | assert_stmt)
expr_stmt: testlist_star_expr (augassign (yield_expr|testlist) |
('=' (yield_expr|testlist_star_expr))*)
testlist_star_expr: (test|star_expr) (',' (test|star_expr))* [',']
augassign: ('+=' | '-=' | '*=' | '/=' | '%=' | '&=' | '|=' | '^=' |
'<<=' | '>>=' | '**=' | '//=')
# For normal assignments, additional restrictions enforced by the interpreter
print_stmt: 'print' ( [ test (',' test)* [','] ] |
'>>' test [ (',' test)+ [','] ] )
del_stmt: 'del' exprlist
pass_stmt: 'pass'
flow_stmt: break_stmt | continue_stmt | return_stmt | raise_stmt | yield_stmt
break_stmt: 'break'
continue_stmt: 'continue'
return_stmt: 'return' [testlist]
yield_stmt: yield_expr
raise_stmt: 'raise' [test ['from' test | ',' test [',' test]]]
import_stmt: import_name | import_from
import_name: 'import' dotted_as_names
# note below: the ('.' | '...') is necessary because '...' is tokenized as ELLIPSIS
import_from: ('from' (('.' | '...')* dotted_name | ('.' | '...')+)
'import' ('*' | '(' import_as_names ')' | import_as_names))
import_as_name: NAME ['as' NAME]
dotted_as_name: dotted_name ['as' NAME]
import_as_names: import_as_name (',' import_as_name)* [',']
dotted_as_names: dotted_as_name (',' dotted_as_name)*
dotted_name: NAME ('.' NAME)*
global_stmt: ('global' | 'nonlocal') NAME (',' NAME)*
exec_stmt: 'exec' expr ['in' test [',' test]]
assert_stmt: 'assert' test [',' test]
compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | with_stmt | funcdef | classdef | decorated
if_stmt: 'if' test ':' suite ('elif' test ':' suite)* ['else' ':' suite]
while_stmt: 'while' test ':' suite ['else' ':' suite]
for_stmt: 'for' exprlist 'in' testlist ':' suite ['else' ':' suite]
try_stmt: ('try' ':' suite
((except_clause ':' suite)+
['else' ':' suite]
['finally' ':' suite] |
'finally' ':' suite))
with_stmt: 'with' with_item (',' with_item)* ':' suite
with_item: test ['as' expr]
with_var: 'as' expr
# NB compile.c makes sure that the default except clause is last
except_clause: 'except' [test [(',' | 'as') test]]
# Edit by David Halter: The stmt is now optional. This reflects how Jedi allows
# classes and functions to be empty, which is beneficial for autocompletion.
suite: simple_stmt | NEWLINE INDENT stmt* DEDENT
# Backward compatibility cruft to support:
# [ x for x in lambda: True, lambda: False if x() ]
# even while also allowing:
# lambda x: 5 if x else 2
# (But not a mix of the two)
testlist_safe: old_test [(',' old_test)+ [',']]
old_test: or_test | old_lambdef
old_lambdef: 'lambda' [varargslist] ':' old_test
test: or_test ['if' or_test 'else' test] | lambdef
or_test: and_test ('or' and_test)*
and_test: not_test ('and' not_test)*
not_test: 'not' not_test | comparison
comparison: expr (comp_op expr)*
comp_op: '<'|'>'|'=='|'>='|'<='|'<>'|'!='|'in'|'not' 'in'|'is'|'is' 'not'
star_expr: '*' expr
expr: xor_expr ('|' xor_expr)*
xor_expr: and_expr ('^' and_expr)*
and_expr: shift_expr ('&' shift_expr)*
shift_expr: arith_expr (('<<'|'>>') arith_expr)*
arith_expr: term (('+'|'-') term)*
term: factor (('*'|'/'|'%'|'//') factor)*
factor: ('+'|'-'|'~') factor | power
power: atom trailer* ['**' factor]
atom: ('(' [yield_expr|testlist_comp] ')' |
'[' [testlist_comp] ']' |
'{' [dictorsetmaker] '}' |
'`' testlist1 '`' |
NAME | NUMBER | STRING+ | '.' '.' '.')
# Modification by David Halter, remove `testlist_gexp` and `listmaker`
testlist_comp: (test|star_expr) ( comp_for | (',' (test|star_expr))* [','] )
lambdef: 'lambda' [varargslist] ':' test
trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME
subscriptlist: subscript (',' subscript)* [',']
subscript: test | [test] ':' [test] [sliceop]
sliceop: ':' [test]
exprlist: (expr|star_expr) (',' (expr|star_expr))* [',']
testlist: test (',' test)* [',']
# Modification by David Halter, dictsetmaker -> dictorsetmaker (so that it's
# the same as in the 3.4 grammar).
dictorsetmaker: ( (test ':' test (comp_for | (',' test ':' test)* [','])) |
(test (comp_for | (',' test)* [','])) )
classdef: 'class' NAME ['(' [arglist] ')'] ':' suite
arglist: (argument ',')* (argument [',']
|'*' test (',' argument)* [',' '**' test]
|'**' test)
argument: test [comp_for] | test '=' test # Really [keyword '='] test
comp_iter: comp_for | comp_if
comp_for: 'for' exprlist 'in' testlist_safe [comp_iter]
comp_if: 'if' old_test [comp_iter]
testlist1: test (',' test)*
# not used in grammar, but may appear in "node" passed from Parser to Compiler
encoding_decl: NAME
yield_expr: 'yield' [testlist]
jedi/parser/grammar3.4.txt
+135
View File
@@ -0,0 +1,135 @@
# Grammar for Python
# Note: Changing the grammar specified in this file will most likely
# require corresponding changes in the parser module
# (../Modules/parsermodule.c). If you can't make the changes to
# that module yourself, please co-ordinate the required changes
# with someone who can; ask around on python-dev for help. Fred
# Drake <fdrake@acm.org> will probably be listening there.
# NOTE WELL: You should also follow all the steps listed in PEP 306,
# "How to Change Python's Grammar"
# Start symbols for the grammar:
# single_input is a single interactive statement;
# file_input is a module or sequence of commands read from an input file;
# eval_input is the input for the eval() functions.
# NB: compound_stmt in single_input is followed by extra NEWLINE!
file_input: (NEWLINE | stmt)* ENDMARKER
single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE
eval_input: testlist NEWLINE* ENDMARKER
decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE
decorators: decorator+
decorated: decorators (classdef | funcdef)
funcdef: 'def' NAME parameters ['->' test] ':' suite
parameters: '(' [typedargslist] ')'
typedargslist: (tfpdef ['=' test] (',' tfpdef ['=' test])* [','
['*' [tfpdef] (',' tfpdef ['=' test])* [',' '**' tfpdef] | '**' tfpdef]]
| '*' [tfpdef] (',' tfpdef ['=' test])* [',' '**' tfpdef] | '**' tfpdef)
tfpdef: NAME [':' test]
varargslist: (vfpdef ['=' test] (',' vfpdef ['=' test])* [','
['*' [vfpdef] (',' vfpdef ['=' test])* [',' '**' vfpdef] | '**' vfpdef]]
| '*' [vfpdef] (',' vfpdef ['=' test])* [',' '**' vfpdef] | '**' vfpdef)
vfpdef: NAME
stmt: simple_stmt | compound_stmt
simple_stmt: small_stmt (';' small_stmt)* [';'] NEWLINE
small_stmt: (expr_stmt | del_stmt | pass_stmt | flow_stmt |
import_stmt | global_stmt | nonlocal_stmt | assert_stmt)
expr_stmt: testlist_star_expr (augassign (yield_expr|testlist) |
('=' (yield_expr|testlist_star_expr))*)
testlist_star_expr: (test|star_expr) (',' (test|star_expr))* [',']
augassign: ('+=' | '-=' | '*=' | '/=' | '%=' | '&=' | '|=' | '^=' |
'<<=' | '>>=' | '**=' | '//=')
# For normal assignments, additional restrictions enforced by the interpreter
del_stmt: 'del' exprlist
pass_stmt: 'pass'
flow_stmt: break_stmt | continue_stmt | return_stmt | raise_stmt | yield_stmt
break_stmt: 'break'
continue_stmt: 'continue'
return_stmt: 'return' [testlist]
yield_stmt: yield_expr
raise_stmt: 'raise' [test ['from' test]]
import_stmt: import_name | import_from
import_name: 'import' dotted_as_names
# note below: the ('.' | '...') is necessary because '...' is tokenized as ELLIPSIS
import_from: ('from' (('.' | '...')* dotted_name | ('.' | '...')+)
'import' ('*' | '(' import_as_names ')' | import_as_names))
import_as_name: NAME ['as' NAME]
dotted_as_name: dotted_name ['as' NAME]
import_as_names: import_as_name (',' import_as_name)* [',']
dotted_as_names: dotted_as_name (',' dotted_as_name)*
dotted_name: NAME ('.' NAME)*
global_stmt: 'global' NAME (',' NAME)*
nonlocal_stmt: 'nonlocal' NAME (',' NAME)*
assert_stmt: 'assert' test [',' test]
compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | with_stmt | funcdef | classdef | decorated
if_stmt: 'if' test ':' suite ('elif' test ':' suite)* ['else' ':' suite]
while_stmt: 'while' test ':' suite ['else' ':' suite]
for_stmt: 'for' exprlist 'in' testlist ':' suite ['else' ':' suite]
try_stmt: ('try' ':' suite
((except_clause ':' suite)+
['else' ':' suite]
['finally' ':' suite] |
'finally' ':' suite))
with_stmt: 'with' with_item (',' with_item)* ':' suite
with_item: test ['as' expr]
# NB compile.c makes sure that the default except clause is last
except_clause: 'except' [test ['as' NAME]]
# Edit by David Halter: The stmt is now optional. This reflects how Jedi allows
# classes and functions to be empty, which is beneficial for autocompletion.
suite: simple_stmt | NEWLINE INDENT stmt* DEDENT
test: or_test ['if' or_test 'else' test] | lambdef
test_nocond: or_test | lambdef_nocond
lambdef: 'lambda' [varargslist] ':' test
lambdef_nocond: 'lambda' [varargslist] ':' test_nocond
or_test: and_test ('or' and_test)*
and_test: not_test ('and' not_test)*
not_test: 'not' not_test | comparison
comparison: expr (comp_op expr)*
# <> isn't actually a valid comparison operator in Python. It's here for the
# sake of a __future__ import described in PEP 401
comp_op: '<'|'>'|'=='|'>='|'<='|'<>'|'!='|'in'|'not' 'in'|'is'|'is' 'not'
star_expr: '*' expr
expr: xor_expr ('|' xor_expr)*
xor_expr: and_expr ('^' and_expr)*
and_expr: shift_expr ('&' shift_expr)*
shift_expr: arith_expr (('<<'|'>>') arith_expr)*
arith_expr: term (('+'|'-') term)*
term: factor (('*'|'/'|'%'|'//') factor)*
factor: ('+'|'-'|'~') factor | power
power: atom trailer* ['**' factor]
atom: ('(' [yield_expr|testlist_comp] ')' |
'[' [testlist_comp] ']' |
'{' [dictorsetmaker] '}' |
NAME | NUMBER | STRING+ | '...' | 'None' | 'True' | 'False')
testlist_comp: (test|star_expr) ( comp_for | (',' (test|star_expr))* [','] )
trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME
subscriptlist: subscript (',' subscript)* [',']
subscript: test | [test] ':' [test] [sliceop]
sliceop: ':' [test]
exprlist: (expr|star_expr) (',' (expr|star_expr))* [',']
testlist: test (',' test)* [',']
dictorsetmaker: ( (test ':' test (comp_for | (',' test ':' test)* [','])) |
(test (comp_for | (',' test)* [','])) )
classdef: 'class' NAME ['(' [arglist] ')'] ':' suite
arglist: (argument ',')* (argument [',']
|'*' test (',' argument)* [',' '**' test]
|'**' test)
# The reason that keywords are test nodes instead of NAME is that using NAME
# results in an ambiguity. ast.c makes sure it's a NAME.
argument: test [comp_for] | test '=' test # Really [keyword '='] test
comp_iter: comp_for | comp_if
comp_for: 'for' exprlist 'in' or_test [comp_iter]
comp_if: 'if' test_nocond [comp_iter]
# not used in grammar, but may appear in "node" passed from Parser to Compiler
encoding_decl: NAME
yield_expr: 'yield' [yield_arg]
yield_arg: 'from' test | testlist
jedi/parser/pgen2/__init__.py
+8
View File
@@ -0,0 +1,8 @@
# Copyright 2004-2005 Elemental Security, Inc. All Rights Reserved.
# Licensed to PSF under a Contributor Agreement.
# Modifications:
# Copyright 2006 Google, Inc. All Rights Reserved.
# Licensed to PSF under a Contributor Agreement.
# Copyright 2014 David Halter. Integration into Jedi.
# Modifications are dual-licensed: MIT and PSF.
jedi/parser/pgen2/grammar.py
+125
View File
@@ -0,0 +1,125 @@
# Copyright 2004-2005 Elemental Security, Inc. All Rights Reserved.
# Licensed to PSF under a Contributor Agreement.
# Modifications:
# Copyright 2014 David Halter. Integration into Jedi.
# Modifications are dual-licensed: MIT and PSF.
"""This module defines the data structures used to represent a grammar.
These are a bit arcane because they are derived from the data
structures used by Python's 'pgen' parser generator.
There's also a table here mapping operators to their names in the
token module; the Python tokenize module reports all operators as the
fallback token code OP, but the parser needs the actual token code.
"""
# Python imports
import pickle
class Grammar(object):
"""Pgen parsing tables conversion class.
Once initialized, this class supplies the grammar tables for the
parsing engine implemented by parse.py. The parsing engine
accesses the instance variables directly. The class here does not
provide initialization of the tables; several subclasses exist to
do this (see the conv and pgen modules).
The load() method reads the tables from a pickle file, which is
much faster than the other ways offered by subclasses. The pickle
file is written by calling dump() (after loading the grammar
tables using a subclass). The report() method prints a readable
representation of the tables to stdout, for debugging.
The instance variables are as follows:
symbol2number -- a dict mapping symbol names to numbers. Symbol
numbers are always 256 or higher, to distinguish
them from token numbers, which are between 0 and
255 (inclusive).
number2symbol -- a dict mapping numbers to symbol names;
these two are each other's inverse.
states -- a list of DFAs, where each DFA is a list of
states, each state is a list of arcs, and each
arc is a (i, j) pair where i is a label and j is
a state number. The DFA number is the index into
this list. (This name is slightly confusing.)
Final states are represented by a special arc of
the form (0, j) where j is its own state number.
dfas -- a dict mapping symbol numbers to (DFA, first)
pairs, where DFA is an item from the states list
above, and first is a set of tokens that can
begin this grammar rule (represented by a dict
whose values are always 1).
labels -- a list of (x, y) pairs where x is either a token
number or a symbol number, and y is either None
or a string; the strings are keywords. The label
number is the index in this list; label numbers
are used to mark state transitions (arcs) in the
DFAs.
start -- the number of the grammar's start symbol.
keywords -- a dict mapping keyword strings to arc labels.
tokens -- a dict mapping token numbers to arc labels.
"""
def __init__(self):
self.symbol2number = {}
self.number2symbol = {}
self.states = []
self.dfas = {}
self.labels = [(0, "EMPTY")]
self.keywords = {}
self.tokens = {}
self.symbol2label = {}
self.start = 256
def dump(self, filename):
"""Dump the grammar tables to a pickle file."""
with open(filename, "wb") as f:
pickle.dump(self.__dict__, f, 2)
def load(self, filename):
"""Load the grammar tables from a pickle file."""
with open(filename, "rb") as f:
d = pickle.load(f)
self.__dict__.update(d)
def copy(self):
"""
Copy the grammar.
"""
new = self.__class__()
for dict_attr in ("symbol2number", "number2symbol", "dfas", "keywords",
"tokens", "symbol2label"):
setattr(new, dict_attr, getattr(self, dict_attr).copy())
new.labels = self.labels[:]
new.states = self.states[:]
new.start = self.start
return new
def report(self):
"""Dump the grammar tables to standard output, for debugging."""
from pprint import pprint
print("s2n")
pprint(self.symbol2number)
print("n2s")
pprint(self.number2symbol)
print("states")
pprint(self.states)
print("dfas")
pprint(self.dfas)
print("labels")
pprint(self.labels)
print("start", self.start)
jedi/parser/pgen2/parse.py
+205
View File
@@ -0,0 +1,205 @@
# Copyright 2004-2005 Elemental Security, Inc. All Rights Reserved.
# Licensed to PSF under a Contributor Agreement.
# Modifications:
# Copyright 2014 David Halter. Integration into Jedi.
# Modifications are dual-licensed: MIT and PSF.
"""
Parser engine for the grammar tables generated by pgen.
The grammar table must be loaded first.
See Parser/parser.c in the Python distribution for additional info on
how this parsing engine works.
"""
# Local imports
from jedi.parser import tokenize
class ParseError(Exception):
"""Exception to signal the parser is stuck."""
def __init__(self, msg, type, value, start_pos):
Exception.__init__(self, "%s: type=%r, value=%r, start_pos=%r" %
(msg, tokenize.tok_name[type], value, start_pos))
self.msg = msg
self.type = type
self.value = value
self.start_pos = start_pos
class PgenParser(object):
"""Parser engine.
The proper usage sequence is:
p = Parser(grammar, [converter]) # create instance
p.setup([start]) # prepare for parsing
<for each input token>:
if p.addtoken(...): # parse a token; may raise ParseError
break
root = p.rootnode # root of abstract syntax tree
A Parser instance may be reused by calling setup() repeatedly.
A Parser instance contains state pertaining to the current token
sequence, and should not be used concurrently by different threads
to parse separate token sequences.
See driver.py for how to get input tokens by tokenizing a file or
string.
Parsing is complete when addtoken() returns True; the root of the
abstract syntax tree can then be retrieved from the rootnode
instance variable. When a syntax error occurs, addtoken() raises
the ParseError exception. There is no error recovery; the parser
cannot be used after a syntax error was reported (but it can be
reinitialized by calling setup()).
"""
def __init__(self, grammar, convert_node, convert_leaf, error_recovery):
"""Constructor.
The grammar argument is a grammar.Grammar instance; see the
grammar module for more information.
The parser is not ready yet for parsing; you must call the
setup() method to get it started.
The optional convert argument is a function mapping concrete
syntax tree nodes to abstract syntax tree nodes. If not
given, no conversion is done and the syntax tree produced is
the concrete syntax tree. If given, it must be a function of
two arguments, the first being the grammar (a grammar.Grammar
instance), and the second being the concrete syntax tree node
to be converted. The syntax tree is converted from the bottom
up.
A concrete syntax tree node is a (type, nodes) tuple, where
type is the node type (a token or symbol number) and nodes
is a list of children for symbols, and None for tokens.
An abstract syntax tree node may be anything; this is entirely
up to the converter function.
"""
self.grammar = grammar
self.convert_node = convert_node
self.convert_leaf = convert_leaf
# Prepare for parsing.
start = self.grammar.start
# Each stack entry is a tuple: (dfa, state, node).
# A node is a tuple: (type, children),
# where children is a list of nodes or None
newnode = (start, [])
stackentry = (self.grammar.dfas[start], 0, newnode)
self.stack = [stackentry]
self.rootnode = None
self.error_recovery = error_recovery
def parse(self, tokenizer):
for type, value, prefix, start_pos in tokenizer:
if self.addtoken(type, value, prefix, start_pos):
break
else:
# We never broke out -- EOF is too soon -- Unfinished statement.
self.error_recovery(self.grammar, self.stack, type, value,
start_pos, prefix, self.addtoken)
# Add the ENDMARKER again.
if not self.addtoken(type, value, prefix, start_pos):
raise ParseError("incomplete input", type, value, start_pos)
return self.rootnode
def addtoken(self, type, value, prefix, start_pos):
"""Add a token; return True if this is the end of the program."""
# Map from token to label
if type == tokenize.NAME:
# Check for reserved words (keywords)
try:
ilabel = self.grammar.keywords[value]
except KeyError:
ilabel = self.grammar.tokens[type]
else:
ilabel = self.grammar.tokens[type]
# Loop until the token is shifted; may raise exceptions
while True:
dfa, state, node = self.stack[-1]
states, first = dfa
arcs = states[state]
# Look for a state with this label
for i, newstate in arcs:
t, v = self.grammar.labels[i]
if ilabel == i:
# Look it up in the list of labels
assert t < 256
# Shift a token; we're done with it
self.shift(type, value, newstate, prefix, start_pos)
# Pop while we are in an accept-only state
state = newstate
while states[state] == [(0, state)]:
self.pop()
if not self.stack:
# Done parsing!
return True
dfa, state, node = self.stack[-1]
states, first = dfa
# Done with this token
return False
elif t >= 256:
# See if it's a symbol and if we're in its first set
itsdfa = self.grammar.dfas[t]
itsstates, itsfirst = itsdfa
if ilabel in itsfirst:
# Push a symbol
self.push(t, itsdfa, newstate)
break # To continue the outer while loop
else:
if (0, state) in arcs:
# An accepting state, pop it and try something else
self.pop()
if not self.stack:
# Done parsing, but another token is input
raise ParseError("too much input", type, value, start_pos)
else:
self.error_recovery(self.grammar, self.stack, type,
value, start_pos, prefix, self.addtoken)
break
def shift(self, type, value, newstate, prefix, start_pos):
"""Shift a token. (Internal)"""
dfa, state, node = self.stack[-1]
newnode = self.convert_leaf(self.grammar, type, value, prefix, start_pos)
node[-1].append(newnode)
self.stack[-1] = (dfa, newstate, node)
def push(self, type, newdfa, newstate):
"""Push a nonterminal. (Internal)"""
dfa, state, node = self.stack[-1]
newnode = (type, [])
self.stack[-1] = (dfa, newstate, node)
self.stack.append((newdfa, 0, newnode))
def pop(self):
"""Pop a nonterminal. (Internal)"""
popdfa, popstate, (type, children) = self.stack.pop()
# If there's exactly one child, return that child instead of creating a
# new node. We still create expr_stmt and file_input though, because a
# lot of Jedi depends on its logic.
if len(children) == 1:
newnode = children[0]
else:
newnode = self.convert_node(self.grammar, type, children)
try:
# Equal to:
# dfa, state, node = self.stack[-1]
# symbol, children = node
self.stack[-1][2][1].append(newnode)
except IndexError:
# Stack is empty, set the rootnode.
self.rootnode = newnode
jedi/parser/pgen2/pgen.py
+394
View File
@@ -0,0 +1,394 @@
# Copyright 2004-2005 Elemental Security, Inc. All Rights Reserved.
# Licensed to PSF under a Contributor Agreement.
# Modifications:
# Copyright 2014 David Halter. Integration into Jedi.
# Modifications are dual-licensed: MIT and PSF.
# Pgen imports
from . import grammar
from jedi.parser import token
from jedi.parser import tokenize
class ParserGenerator(object):
def __init__(self, filename, stream=None):
close_stream = None
if stream is None:
stream = open(filename)
close_stream = stream.close
self.filename = filename
self.stream = stream
self.generator = tokenize.generate_tokens(stream.readline)
self.gettoken() # Initialize lookahead
self.dfas, self.startsymbol = self.parse()
if close_stream is not None:
close_stream()
self.first = {} # map from symbol name to set of tokens
self.addfirstsets()
def make_grammar(self):
c = grammar.Grammar()
names = list(self.dfas.keys())
names.sort()
names.remove(self.startsymbol)
names.insert(0, self.startsymbol)
for name in names:
i = 256 + len(c.symbol2number)
c.symbol2number[name] = i
c.number2symbol[i] = name
for name in names:
dfa = self.dfas[name]
states = []
for state in dfa:
arcs = []
for label, next in state.arcs.items():
arcs.append((self.make_label(c, label), dfa.index(next)))
if state.isfinal:
arcs.append((0, dfa.index(state)))
states.append(arcs)
c.states.append(states)
c.dfas[c.symbol2number[name]] = (states, self.make_first(c, name))
c.start = c.symbol2number[self.startsymbol]
return c
def make_first(self, c, name):
rawfirst = self.first[name]
first = {}
for label in rawfirst:
ilabel = self.make_label(c, label)
##assert ilabel not in first # XXX failed on <> ... !=
first[ilabel] = 1
return first
def make_label(self, c, label):
# XXX Maybe this should be a method on a subclass of converter?
ilabel = len(c.labels)
if label[0].isalpha():
# Either a symbol name or a named token
if label in c.symbol2number:
# A symbol name (a non-terminal)
if label in c.symbol2label:
return c.symbol2label[label]
else:
c.labels.append((c.symbol2number[label], None))
c.symbol2label[label] = ilabel
return ilabel
else:
# A named token (NAME, NUMBER, STRING)
itoken = getattr(token, label, None)
assert isinstance(itoken, int), label
assert itoken in token.tok_name, label
if itoken in c.tokens:
return c.tokens[itoken]
else:
c.labels.append((itoken, None))
c.tokens[itoken] = ilabel
return ilabel
else:
# Either a keyword or an operator
assert label[0] in ('"', "'"), label
value = eval(label)
if value[0].isalpha():
# A keyword
if value in c.keywords:
return c.keywords[value]
else:
c.labels.append((token.NAME, value))
c.keywords[value] = ilabel
return ilabel
else:
# An operator (any non-numeric token)
itoken = token.opmap[value] # Fails if unknown token
if itoken in c.tokens:
return c.tokens[itoken]
else:
c.labels.append((itoken, None))
c.tokens[itoken] = ilabel
return ilabel
def addfirstsets(self):
names = list(self.dfas.keys())
names.sort()
for name in names:
if name not in self.first:
self.calcfirst(name)
#print name, self.first[name].keys()
def calcfirst(self, name):
dfa = self.dfas[name]
self.first[name] = None # dummy to detect left recursion
state = dfa[0]
totalset = {}
overlapcheck = {}
for label, next in state.arcs.items():
if label in self.dfas:
if label in self.first:
fset = self.first[label]
if fset is None:
raise ValueError("recursion for rule %r" % name)
else:
self.calcfirst(label)
fset = self.first[label]
totalset.update(fset)
overlapcheck[label] = fset
else:
totalset[label] = 1
overlapcheck[label] = {label: 1}
inverse = {}
for label, itsfirst in overlapcheck.items():
for symbol in itsfirst:
if symbol in inverse:
raise ValueError("rule %s is ambiguous; %s is in the"
" first sets of %s as well as %s" %
(name, symbol, label, inverse[symbol]))
inverse[symbol] = label
self.first[name] = totalset
def parse(self):
dfas = {}
startsymbol = None
# MSTART: (NEWLINE | RULE)* ENDMARKER
while self.type != token.ENDMARKER:
while self.type == token.NEWLINE:
self.gettoken()
# RULE: NAME ':' RHS NEWLINE
name = self.expect(token.NAME)
self.expect(token.OP, ":")
a, z = self.parse_rhs()
self.expect(token.NEWLINE)
#self.dump_nfa(name, a, z)
dfa = self.make_dfa(a, z)
#self.dump_dfa(name, dfa)
# oldlen = len(dfa)
self.simplify_dfa(dfa)
# newlen = len(dfa)
dfas[name] = dfa
#print name, oldlen, newlen
if startsymbol is None:
startsymbol = name
return dfas, startsymbol
def make_dfa(self, start, finish):
# To turn an NFA into a DFA, we define the states of the DFA
# to correspond to *sets* of states of the NFA. Then do some
# state reduction. Let's represent sets as dicts with 1 for
# values.
assert isinstance(start, NFAState)
assert isinstance(finish, NFAState)
def closure(state):
base = {}
addclosure(state, base)
return base
def addclosure(state, base):
assert isinstance(state, NFAState)
if state in base:
return
base[state] = 1
for label, next in state.arcs:
if label is None:
addclosure(next, base)
states = [DFAState(closure(start), finish)]
for state in states: # NB states grows while we're iterating
arcs = {}
for nfastate in state.nfaset:
for label, next in nfastate.arcs:
if label is not None:
addclosure(next, arcs.setdefault(label, {}))
for label, nfaset in arcs.items():
for st in states:
if st.nfaset == nfaset:
break
else:
st = DFAState(nfaset, finish)
states.append(st)
state.addarc(st, label)
return states # List of DFAState instances; first one is start
def dump_nfa(self, name, start, finish):
print("Dump of NFA for", name)
todo = [start]
for i, state in enumerate(todo):
print(" State", i, state is finish and "(final)" or "")
for label, next in state.arcs:
if next in todo:
j = todo.index(next)
else:
j = len(todo)
todo.append(next)
if label is None:
print(" -> %d" % j)
else:
print(" %s -> %d" % (label, j))
def dump_dfa(self, name, dfa):
print("Dump of DFA for", name)
for i, state in enumerate(dfa):
print(" State", i, state.isfinal and "(final)" or "")
for label, next in state.arcs.items():
print(" %s -> %d" % (label, dfa.index(next)))
def simplify_dfa(self, dfa):
# This is not theoretically optimal, but works well enough.
# Algorithm: repeatedly look for two states that have the same
# set of arcs (same labels pointing to the same nodes) and
# unify them, until things stop changing.
# dfa is a list of DFAState instances
changes = True
while changes:
changes = False
for i, state_i in enumerate(dfa):
for j in range(i + 1, len(dfa)):
state_j = dfa[j]
if state_i == state_j:
#print " unify", i, j
del dfa[j]
for state in dfa:
state.unifystate(state_j, state_i)
changes = True
break
def parse_rhs(self):
# RHS: ALT ('|' ALT)*
a, z = self.parse_alt()
if self.value != "|":
return a, z
else:
aa = NFAState()
zz = NFAState()
aa.addarc(a)
z.addarc(zz)
while self.value == "|":
self.gettoken()
a, z = self.parse_alt()
aa.addarc(a)
z.addarc(zz)
return aa, zz
def parse_alt(self):
# ALT: ITEM+
a, b = self.parse_item()
while (self.value in ("(", "[") or
self.type in (token.NAME, token.STRING)):
c, d = self.parse_item()
b.addarc(c)
b = d
return a, b
def parse_item(self):
# ITEM: '[' RHS ']' | ATOM ['+' | '*']
if self.value == "[":
self.gettoken()
a, z = self.parse_rhs()
self.expect(token.OP, "]")
a.addarc(z)
return a, z
else:
a, z = self.parse_atom()
value = self.value
if value not in ("+", "*"):
return a, z
self.gettoken()
z.addarc(a)
if value == "+":
return a, z
else:
return a, a
def parse_atom(self):
# ATOM: '(' RHS ')' | NAME | STRING
if self.value == "(":
self.gettoken()
a, z = self.parse_rhs()
self.expect(token.OP, ")")
return a, z
elif self.type in (token.NAME, token.STRING):
a = NFAState()
z = NFAState()
a.addarc(z, self.value)
self.gettoken()
return a, z
else:
self.raise_error("expected (...) or NAME or STRING, got %s/%s",
self.type, self.value)
def expect(self, type, value=None):
if self.type != type or (value is not None and self.value != value):
self.raise_error("expected %s/%s, got %s/%s",
type, value, self.type, self.value)
value = self.value
self.gettoken()
return value
def gettoken(self):
tup = next(self.generator)
while tup[0] in (token.COMMENT, token.NL):
tup = next(self.generator)
self.type, self.value, self.begin, prefix = tup
#print tokenize.tok_name[self.type], repr(self.value)
def raise_error(self, msg, *args):
if args:
try:
msg = msg % args
except:
msg = " ".join([msg] + list(map(str, args)))
line = open(self.filename).readlines()[self.begin[0]]
raise SyntaxError(msg, (self.filename, self.begin[0],
self.begin[1], line))
class NFAState(object):
def __init__(self):
self.arcs = [] # list of (label, NFAState) pairs
def addarc(self, next, label=None):
assert label is None or isinstance(label, str)
assert isinstance(next, NFAState)
self.arcs.append((label, next))
class DFAState(object):
def __init__(self, nfaset, final):
assert isinstance(nfaset, dict)
assert isinstance(next(iter(nfaset)), NFAState)
assert isinstance(final, NFAState)
self.nfaset = nfaset
self.isfinal = final in nfaset
self.arcs = {} # map from label to DFAState
def addarc(self, next, label):
assert isinstance(label, str)
assert label not in self.arcs
assert isinstance(next, DFAState)
self.arcs[label] = next
def unifystate(self, old, new):
for label, next in self.arcs.items():
if next is old:
self.arcs[label] = new
def __eq__(self, other):
# Equality test -- ignore the nfaset instance variable
assert isinstance(other, DFAState)
if self.isfinal != other.isfinal:
return False
# Can't just return self.arcs == other.arcs, because that
# would invoke this method recursively, with cycles...
if len(self.arcs) != len(other.arcs):
return False
for label, next in self.arcs.items():
if next is not other.arcs.get(label):
return False
return True
__hash__ = None # For Py3 compatibility.
def generate_grammar(filename="Grammar.txt"):
p = ParserGenerator(filename)
return p.make_grammar()
jedi/parser/representation.py
-1614
View File
File diff suppressed because it is too large Load Diff
jedi/parser/token.py
+84
View File
@@ -0,0 +1,84 @@
from __future__ import absolute_import
from jedi._compatibility import is_py3
from token import *
COMMENT = N_TOKENS
tok_name[COMMENT] = 'COMMENT'
N_TOKENS += 1
NL = N_TOKENS
tok_name[NL] = 'NL'
N_TOKENS += 1
if is_py3:
BACKQUOTE = N_TOKENS
tok_name[BACKQUOTE] = 'BACKQUOTE'
N_TOKENS += 1
else:
RARROW = N_TOKENS
tok_name[RARROW] = 'RARROW'
N_TOKENS += 1
ELLIPSIS = N_TOKENS
tok_name[ELLIPSIS] = 'ELLIPSIS'
N_TOKENS += 1
# Map from operator to number (since tokenize doesn't do this)
opmap_raw = """\
( LPAR
) RPAR
[ LSQB
] RSQB
: COLON
, COMMA
; SEMI
+ PLUS
- MINUS
* STAR
/ SLASH
| VBAR
& AMPER
< LESS
> GREATER
= EQUAL
. DOT
% PERCENT
` BACKQUOTE
{ LBRACE
} RBRACE
@ AT
== EQEQUAL
!= NOTEQUAL
<> NOTEQUAL
<= LESSEQUAL
>= GREATEREQUAL
~ TILDE
^ CIRCUMFLEX
<< LEFTSHIFT
>> RIGHTSHIFT
** DOUBLESTAR
+= PLUSEQUAL
-= MINEQUAL
*= STAREQUAL
/= SLASHEQUAL
%= PERCENTEQUAL
&= AMPEREQUAL
|= VBAREQUAL
^= CIRCUMFLEXEQUAL
<<= LEFTSHIFTEQUAL
>>= RIGHTSHIFTEQUAL
**= DOUBLESTAREQUAL
// DOUBLESLASH
//= DOUBLESLASHEQUAL
-> RARROW
... ELLIPSIS
"""
opmap = {}
for line in opmap_raw.splitlines():
op, name = line.split()
opmap[op] = globals()[name]
jedi/parser/tokenize.py
+84 -104
View File
@@ -14,95 +14,26 @@ from __future__ import absolute_import
import string
import re
from io import StringIO
from token import (tok_name, N_TOKENS, ENDMARKER, STRING, NUMBER, NAME, OP,
ERRORTOKEN, NEWLINE)
from jedi.parser.token import (tok_name, N_TOKENS, ENDMARKER, STRING, NUMBER,
NAME, OP, ERRORTOKEN, NEWLINE, INDENT, DEDENT)
from jedi._compatibility import is_py3
from jedi._compatibility import u
cookie_re = re.compile("coding[:=]\s*([-\w.]+)")
# From here on we have custom stuff (everything before was originally Python
# internal code).
FLOWS = ['if', 'else', 'elif', 'while', 'with', 'try', 'except', 'finally']
namechars = string.ascii_letters + '_'
if is_py3:
# Python 3 has str.isidentifier() to check if a char is a valid identifier
is_identifier = str.isidentifier
else:
namechars = string.ascii_letters + '_'
is_identifier = lambda s: s in namechars
COMMENT = N_TOKENS
tok_name[COMMENT] = 'COMMENT'
class Token(object):
"""
The token object is an efficient representation of the structure
(type, token, (start_pos_line, start_pos_col)). It has indexer
methods that maintain compatibility to existing code that expects the above
structure.
>>> repr(Token(1, "test", (1, 1)))
"<Token: ('NAME', 'test', (1, 1))>"
>>> Token(1, 'bar', (3, 4)).__getstate__()
(1, 'bar', 3, 4)
>>> a = Token(0, 'baz', (0, 0))
>>> a.__setstate__((1, 'foo', 3, 4))
>>> a
<Token: ('NAME', 'foo', (3, 4))>
>>> a.start_pos
(3, 4)
>>> a.string
'foo'
>>> a._start_pos_col
4
>>> Token(1, u("😷"), (1 ,1)).string + "p" == u("😷p")
True
"""
__slots__ = ("type", "string", "_start_pos_line", "_start_pos_col")
def __init__(self, type, string, start_pos):
self.type = type
self.string = string
self._start_pos_line = start_pos[0]
self._start_pos_col = start_pos[1]
def __repr__(self):
typ = tok_name[self.type]
content = typ, self.string, (self._start_pos_line, self._start_pos_col)
return "<%s: %s>" % (type(self).__name__, content)
@property
def start_pos(self):
return (self._start_pos_line, self._start_pos_col)
@property
def end_pos(self):
"""Returns end position respecting multiline tokens."""
end_pos_line = self._start_pos_line
lines = self.string.split('\n')
if self.string.endswith('\n'):
lines = lines[:-1]
lines[-1] += '\n'
end_pos_line += len(lines) - 1
end_pos_col = self._start_pos_col
# Check for multiline token
if self._start_pos_line == end_pos_line:
end_pos_col += len(lines[-1])
else:
end_pos_col = len(lines[-1])
return (end_pos_line, end_pos_col)
# Make cache footprint smaller for faster unpickling
def __getstate__(self):
return (self.type, self.string, self._start_pos_line, self._start_pos_col)
def __setstate__(self, state):
self.type = state[0]
self.string = state[1]
self._start_pos_line = state[2]
self._start_pos_col = state[3]
def group(*choices):
return '(' + '|'.join(choices) + ')'
@@ -158,7 +89,8 @@ cont_str = group(r"[bBuU]?[rR]?'[^\n'\\]*(?:\\.[^\n'\\]*)*" +
r'[bBuU]?[rR]?"[^\n"\\]*(?:\\.[^\n"\\]*)*' +
group('"', r'\\\r?\n'))
pseudo_extras = group(r'\\\r?\n', comment, triple)
pseudo_token = whitespace + group(pseudo_extras, number, funny, cont_str, name)
pseudo_token = group(whitespace) + \
group(pseudo_extras, number, funny, cont_str, name)
def _compile(expr):
@@ -167,6 +99,7 @@ def _compile(expr):
pseudoprog, single3prog, double3prog = map(
_compile, (pseudo_token, single3, double3))
endprogs = {"'": _compile(single), '"': _compile(double),
"'''": single3prog, '"""': double3prog,
"r'''": single3prog, 'r"""': double3prog,
@@ -202,28 +135,43 @@ del _compile
tabsize = 8
ALWAYS_BREAK_TOKENS = (';', 'import', 'from', 'class', 'def', 'try', 'except',
'finally', 'while', 'return')
def source_tokens(source, line_offset=0):
def source_tokens(source):
"""Generate tokens from a the source code (string)."""
source = source + '\n' # end with \n, because the parser needs it
readline = StringIO(source).readline
return generate_tokens(readline, line_offset)
return generate_tokens(readline)
def generate_tokens(readline, line_offset=0):
def generate_tokens(readline):
"""
The original stdlib Python version with minor modifications.
Modified to not care about dedents.
A heavily modified Python standard library tokenizer.
Additionally to the default information, yields also the prefix of each
token. This idea comes from lib2to3. The prefix contains all information
that is irrelevant for the parser like newlines in parentheses or comments.
"""
lnum = line_offset
paren_level = 0 # count parentheses
indents = [0]
lnum = 0
numchars = '0123456789'
contstr = ''
contline = None
while True: # loop over lines in stream
line = readline() # readline returns empty if it's finished. See StringIO
# We start with a newline. This makes indent at the first position
# possible. It's not valid Python, but still better than an INDENT in the
# second line (and not in the first). This makes quite a few things in
# Jedi's fast parser possible.
new_line = True
prefix = '' # Should never be required, but here for safety
additional_prefix = ''
while True: # loop over lines in stream
line = readline() # readline returns empty when finished. See StringIO
if not line:
if contstr:
yield Token(ERRORTOKEN, contstr, contstr_start)
yield ERRORTOKEN, contstr, contstr_start, prefix
break
lnum += 1
@@ -233,7 +181,7 @@ def generate_tokens(readline, line_offset=0):
endmatch = endprog.match(line)
if endmatch:
pos = endmatch.end(0)
yield Token(STRING, contstr + line[:pos], contstr_start)
yield STRING, contstr + line[:pos], contstr_start, prefix
contstr = ''
contline = None
else:
@@ -248,32 +196,48 @@ def generate_tokens(readline, line_offset=0):
if line[pos] in '"\'':
# If a literal starts but doesn't end the whole rest of the
# line is an error token.
txt = txt = line[pos:]
yield Token(ERRORTOKEN, txt, (lnum, pos))
txt = line[pos:]
yield ERRORTOKEN, txt, (lnum, pos), prefix
pos += 1
continue
start, pos = pseudomatch.span(1)
prefix = additional_prefix + pseudomatch.group(1)
additional_prefix = ''
start, pos = pseudomatch.span(2)
spos = (lnum, start)
token, initial = line[start:pos], line[start]
if new_line and initial not in '\r\n#':
new_line = False
if paren_level == 0:
if start > indents[-1]:
yield INDENT, '', spos, ''
indents.append(start)
while start < indents[-1]:
yield DEDENT, '', spos, ''
indents.pop()
if (initial in numchars or # ordinary number
(initial == '.' and token != '.' and token != '...')):
yield Token(NUMBER, token, spos)
yield NUMBER, token, spos, prefix
elif initial in '\r\n':
yield Token(NEWLINE, token, spos)
elif initial == '#':
if not new_line and paren_level == 0:
yield NEWLINE, token, spos, prefix
else:
additional_prefix = prefix + token
new_line = True
elif initial == '#': # Comments
assert not token.endswith("\n")
yield Token(COMMENT, token, spos)
additional_prefix = prefix + token
elif token in triple_quoted:
endprog = endprogs[token]
endmatch = endprog.match(line, pos)
if endmatch: # all on one line
pos = endmatch.end(0)
token = line[start:pos]
yield Token(STRING, token, spos)
yield STRING, token, spos, prefix
else:
contstr_start = (lnum, start) # multiple lines
contstr_start = (lnum, start) # multiple lines
contstr = line[start:]
contline = line
break
@@ -288,12 +252,28 @@ def generate_tokens(readline, line_offset=0):
contline = line
break
else: # ordinary string
yield Token(STRING, token, spos)
elif initial in namechars: # ordinary name
yield Token(NAME, token, spos)
yield STRING, token, spos, prefix
elif is_identifier(initial): # ordinary name
if token in ALWAYS_BREAK_TOKENS:
paren_level = 0
while True:
indent = indents.pop()
if indent > start:
yield DEDENT, '', spos, ''
else:
indents.append(indent)
break
yield NAME, token, spos, prefix
elif initial == '\\' and line[start:] == '\\\n': # continued stmt
continue
additional_prefix += prefix + line[start:]
break
else:
yield Token(OP, token, spos)
if token in '([{':
paren_level += 1
elif token in ')]}':
paren_level -= 1
yield OP, token, spos, prefix
yield Token(ENDMARKER, '', (lnum, 0))
for indent in indents[1:]:
yield DEDENT, '', spos, ''
yield ENDMARKER, '', spos, prefix
jedi/parser/tree.py
+1205
View File
File diff suppressed because it is too large Load Diff
jedi/parser/user_context.py
+138 -61
View File
@@ -1,16 +1,23 @@
import re
import os
import keyword
from jedi import cache
from jedi import common
from jedi.parser import tokenize
from jedi.parser import tokenize, Parser
from jedi._compatibility import u
from jedi.parser.fast import FastParser
from jedi.parser import representation
from jedi.parser import tree as pr
from jedi import debug
from jedi.common import PushBackIterator
REPLACE_STR = r"[bBuU]?[rR]?" + (r"(?:(')[^\n'\\]*(?:\\.[^\n'\\]*)*(?:'|$)" +
'|' +
r'(")[^\n"\\]*(?:\\.[^\n"\\]*)*(?:"|$))')
REPLACE_STR = re.compile(REPLACE_STR)
class UserContext(object):
"""
:param source: The source code of the file.
@@ -22,8 +29,6 @@ class UserContext(object):
self.position = position
self._line_cache = None
# this two are only used, because there is no nonlocal in Python 2
self._line_temp = None
self._relevant_temp = None
@cache.underscore_memoization
@@ -32,62 +37,64 @@ class UserContext(object):
path, self._start_cursor_pos = self._calc_path_until_cursor(self.position)
return path
def _calc_path_until_cursor(self, start_pos=None):
def _backwards_line_generator(self, start_pos):
self._line_temp, self._column_temp = start_pos
first_line = self.get_line(start_pos[0])[:self._column_temp]
self._line_length = self._column_temp
yield first_line[::-1] + '\n'
while True:
self._line_temp -= 1
line = self.get_line(self._line_temp)
self._line_length = len(line)
yield line[::-1] + '\n'
def _get_backwards_tokenizer(self, start_pos, line_gen=None):
if line_gen is None:
line_gen = self._backwards_line_generator(start_pos)
token_gen = tokenize.generate_tokens(lambda: next(line_gen))
for typ, tok_str, tok_start_pos, prefix in token_gen:
line = self.get_line(self._line_temp)
# Calculate the real start_pos of the token.
if tok_start_pos[0] == 1:
# We are in the first checked line
column = start_pos[1] - tok_start_pos[1]
else:
column = len(line) - tok_start_pos[1]
# Multi-line docstrings must be accounted for.
first_line = common.splitlines(tok_str)[0]
column -= len(first_line)
# Reverse the token again, so that it is in normal order again.
yield typ, tok_str[::-1], (self._line_temp, column), prefix[::-1]
def _calc_path_until_cursor(self, start_pos):
"""
Something like a reverse tokenizer that tokenizes the reversed strings.
"""
def fetch_line():
if self._is_first:
self._is_first = False
self._line_length = self._column_temp
line = first_line
else:
line = self.get_line(self._line_temp)
self._line_length = len(line)
line = '\n' + line
# add lines with a backslash at the end
while True:
self._line_temp -= 1
last_line = self.get_line(self._line_temp)
if last_line and last_line[-1] == '\\':
line = last_line[:-1] + ' ' + line
self._line_length = len(last_line)
else:
break
return line[::-1]
self._is_first = True
self._line_temp, self._column_temp = start_cursor = start_pos
first_line = self.get_line(self._line_temp)[:self._column_temp]
open_brackets = ['(', '[', '{']
close_brackets = [')', ']', '}']
gen = PushBackIterator(tokenize.generate_tokens(fetch_line))
start_cursor = start_pos
gen = PushBackIterator(self._get_backwards_tokenizer(start_pos))
string = u('')
level = 0
force_point = False
last_type = None
is_first = True
for tok in gen:
tok_type = tok.type
tok_str = tok.string
end = tok.end_pos
self._column_temp = self._line_length - end[1]
for tok_type, tok_str, tok_start_pos, prefix in gen:
if is_first:
if tok.start_pos != (1, 0): # whitespace is not a path
if prefix: # whitespace is not a path
return u(''), start_cursor
is_first = False
# print 'tok', token_type, tok_str, force_point
if last_type == tok_type == tokenize.NAME:
string += ' '
string = ' ' + string
if level > 0:
if level:
if tok_str in close_brackets:
level += 1
if tok_str in open_brackets:
elif tok_str in open_brackets:
level -= 1
elif tok_str == '.':
force_point = False
@@ -96,7 +103,7 @@ class UserContext(object):
# floating point number.
# The same is true for string prefixes -> represented as a
# combination of string and name.
if tok_type == tokenize.NUMBER and tok_str[0] == '.' \
if tok_type == tokenize.NUMBER and tok_str[-1] == '.' \
or tok_type == tokenize.NAME and last_type == tokenize.STRING:
force_point = False
else:
@@ -104,28 +111,29 @@ class UserContext(object):
elif tok_str in close_brackets:
level += 1
elif tok_type in [tokenize.NAME, tokenize.STRING]:
if keyword.iskeyword(tok_str) and string:
# If there's already something in the string, a keyword
# never adds any meaning to the current statement.
break
force_point = True
elif tok_type == tokenize.NUMBER:
pass
else:
if tok_str == '-':
next_tok = next(gen)
if next_tok.string == 'e':
if next_tok[1] == 'e':
gen.push_back(next_tok)
else:
break
else:
break
x = start_pos[0] - end[0] + 1
l = self.get_line(x)
l = first_line if x == start_pos[0] else l
start_cursor = x, len(l) - end[1]
string += tok_str
start_cursor = tok_start_pos
string = tok_str + prefix + string
last_type = tok_type
# string can still contain spaces at the end
return string[::-1].strip(), start_cursor
# Don't need whitespace around a statement.
return string.strip(), start_cursor
def get_path_under_cursor(self):
"""
@@ -145,6 +153,62 @@ class UserContext(object):
return (before.group(0) if before is not None else '') \
+ (after.group(0) if after is not None else '')
def call_signature(self):
"""
:return: Tuple of string of the call and the index of the cursor.
"""
def get_line(pos):
def simplify_str(match):
"""
To avoid having strings without end marks (error tokens) and
strings that just screw up all the call signatures, just
simplify everything.
"""
mark = match.group(1) or match.group(2)
return mark + ' ' * (len(match.group(0)) - 2) + mark
line_gen = self._backwards_line_generator(pos)
for line in line_gen:
# We have to switch the already backwards lines twice, because
# we scan them from start.
line = line[::-1]
modified = re.sub(REPLACE_STR, simplify_str, line)
yield modified[::-1]
index = 0
level = 0
next_must_be_name = False
next_is_key = False
key_name = None
generator = self._get_backwards_tokenizer(self.position, get_line(self.position))
for tok_type, tok_str, start_pos, prefix in generator:
if tok_str in tokenize.ALWAYS_BREAK_TOKENS:
break
elif next_must_be_name:
if tok_type == tokenize.NAME:
end_pos = start_pos[0], start_pos[1] + len(tok_str)
call, start_pos = self._calc_path_until_cursor(start_pos=end_pos)
return call, index, key_name, start_pos
index = 0
next_must_be_name = False
elif next_is_key:
if tok_type == tokenize.NAME:
key_name = tok_str
next_is_key = False
if tok_str == '(':
level += 1
if level == 1:
next_must_be_name = True
level = 0
elif tok_str == ')':
level -= 1
elif tok_str == ',':
index += 1
elif tok_str == '=':
next_is_key = True
return None, 0, None, (0, 0)
def get_context(self, yield_positions=False):
self.get_path_until_cursor() # In case _start_cursor_pos is undefined.
pos = self._start_cursor_pos
@@ -197,25 +261,31 @@ class UserContext(object):
class UserContextParser(object):
def __init__(self, source, path, position, user_context):
def __init__(self, grammar, source, path, position, user_context,
use_fast_parser=True):
self._grammar = grammar
self._source = source
self._path = path and os.path.abspath(path)
self._position = position
self._user_context = user_context
self._use_fast_parser = use_fast_parser
@cache.underscore_memoization
def _parser(self):
cache.invalidate_star_import_cache(self._path)
parser = FastParser(self._source, self._path)
# Don't pickle that module, because the main module is changing quickly
cache.save_parser(self._path, None, parser, pickling=False)
if self._use_fast_parser:
parser = FastParser(self._grammar, self._source, self._path)
# Don't pickle that module, because the main module is changing quickly
cache.save_parser(self._path, None, parser, pickling=False)
else:
parser = Parser(self._grammar, self._source, self._path)
return parser
@cache.underscore_memoization
def user_stmt(self):
module = self.module()
debug.speed('parsed')
return module.get_statement_for_position(self._position, include_imports=True)
return module.get_statement_for_position(self._position)
@cache.underscore_memoization
def user_stmt_with_whitespace(self):
@@ -234,22 +304,29 @@ class UserContextParser(object):
debug.warning('No statement under the cursor.')
return
pos = next(self._user_context.get_context(yield_positions=True))
user_stmt = self.module().get_statement_for_position(pos, include_imports=True)
user_stmt = self.module().get_statement_for_position(pos)
return user_stmt
@cache.underscore_memoization
def user_scope(self):
"""
Returns the scope in which the user resides. This includes flows.
"""
user_stmt = self.user_stmt()
if user_stmt is None:
def scan(scope):
for s in scope.statements + scope.subscopes:
if isinstance(s, representation.Scope):
if s.start_pos <= self._position <= s.end_pos:
return scan(s) or s
for s in scope.children:
if s.start_pos <= self._position <= s.end_pos:
if isinstance(s, (pr.Scope, pr.Flow)):
if isinstance(s, pr.Flow):
return s
return scan(s) or s
elif s.type in ('suite', 'decorated'):
return scan(s)
return scan(self.module()) or self.module()
else:
return user_stmt.parent
return user_stmt.get_parent_scope(include_flows=True)
def module(self):
return self._parser().module