VimPlug/jedi
1
0
Fork 1
mirror of https://github.com/davidhalter/jedi.git synced 2025-12-07 22:44:27 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

many bugfixes related to fast_parser

Browse Source
This commit is contained in:
David Halter
2012-12-15 16:38:21 +01:00
parent b8b4a02398
commit 2b579eb09b
8 changed files with 95 additions and 52 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
jedi/builtin.py
View File

@@ -9,9 +9,11 @@ if is_py3k:
import types
import inspect
import settings
import common
import debug
import parsing
import fast_parser
import imports
import evaluate
@@ -67,7 +69,11 @@ class CachedModule(object):
def _load_module(self):
source = self._get_source()
self._parser = parsing.PyFuzzyParser(source, self.path or self.name)
p = self.path or self.name
if settings.fast_parser:
self._parser = fast_parser.FastParser(source, p)
else:
self._parser = parsing.PyFuzzyParser(source, p)
p_time = None if not self.path else os.path.getmtime(self.path)
if self.path or self.name:

2
jedi/common.py
View File

@@ -24,7 +24,7 @@ class MultiLevelAttributeError(Exception):
:param base: return values of sys.exc_info().
"""
def __init__(self, base):
def __init__(self, base=None):
self.base = base
def __str__(self):

11
jedi/evaluate.py
View File

@@ -889,10 +889,10 @@ def get_names_for_scope(scope, position=None, star_search=True,
# Ignore the Flows, because the classes and functions care for that.
# InstanceElement of Class is ignored, if it is not the start scope.
if not (scope != non_flow and scope.isinstance(parsing.Class)
or scope.isinstance(parsing.Flow)
or scope.isinstance(Instance) and non_flow.isinstance(Function)
or isinstance(scope, parsing.SubModule) and scope.parent
):
or scope.isinstance(parsing.Flow)
or scope.isinstance(Instance)
and non_flow.isinstance(Function)
):
try:
if isinstance(scope, Instance):
for g in scope.scope_generator():
@@ -1100,9 +1100,6 @@ def get_scopes_for_name(scope, name_str, position=None, search_global=False,
if isinstance(p, InstanceElement) \
and isinstance(p.var, parsing.Class):
p = p.var
if isinstance(p, parsing.SubModule) and p.parent is not None:
p = p.parent
if name_str == name.get_code() and p not in break_scopes:
r, no_break_scope = process(name)
if is_goto:

32
jedi/fast_parser.py
View File

@@ -13,6 +13,7 @@ class Module(parsing.Simple, parsing.Module):
def __init__(self, parsers):
super(Module, self).__init__((1,0))
self.parsers = parsers
self._end_pos = None, None
self.reset_caches()
def reset_caches(self):
@@ -47,7 +48,7 @@ class Module(parsing.Simple, parsing.Module):
elif name in properties:
return self._get(name, properties[name])
else:
raise AttributeError()
raise AttributeError("__getattr__ doesn't offer %s" % name)
def get_statement_for_position(self, pos):
key = 'get_statement_for_position', pos
@@ -63,6 +64,8 @@ class Module(parsing.Simple, parsing.Module):
@property
def used_names(self):
if not self.parsers:
raise NotImplementedError("Parser doesn't exist.")
key = 'used_names'
if key not in self.cache:
dct = {}
@@ -78,27 +81,36 @@ class Module(parsing.Simple, parsing.Module):
@property
def docstr(self):
if not self.parsers:
raise NotImplementedError("Parser doesn't exist.")
return self.parsers[0].module.docstr
@property
def name(self):
if not self.parsers:
raise NotImplementedError("Parser doesn't exist.")
return self.parsers[0].module.name
@property
def path(self):
if not self.parsers:
raise NotImplementedError("Parser doesn't exist.")
return self.parsers[0].module.path
@property
def is_builtin(self):
if not self.parsers:
raise NotImplementedError("Parser doesn't exist.")
return self.parsers[0].module.is_builtin
@property
def end_pos(self):
return self.parsers[-1].module.end_pos
return self._end_pos
@end_pos.setter
def end_pos(self, value):
pass # just ignore, end_pos is not important
if None not in value and self._end_pos < value:
self._end_pos = value
def __repr__(self):
return "<%s: %s@%s-%s>" % (type(self).__name__, self.name,
@@ -153,17 +165,23 @@ class FastParser(use_metaclass(CachedFastParser)):
def update(self, code, user_position=None):
self.user_position = user_position
self.reset_caches()
self.parsers = [] # TODO remove
self.parsers = []
self.module.parsers = self.parsers
self._parse(code)
def _parse(self, code):
parts = re.findall(r'(?:\n(?:def|class)|^).*?(?=\n(?:def|class)|$)',
code, re.DOTALL)
""" :type code: str """
r = r'(?:\n(?:def|class|@.*?\n(?:def|class))|^).*?(?=\n(?:def|class|@)|$)'
parts = re.findall(r, code, re.DOTALL)
line_offset = 0
for p in parts:
lines = p.count('\n')
p = parsing.PyFuzzyParser(p, self.module_path, self.user_position,
line_offset=line_offset, stop_on_scope=True)
line_offset=line_offset, stop_on_scope=True,
top_module=self.module)
p.module.parent = self.module
line_offset += lines
self.parsers.append(p)

2
jedi/helpers.py
View File

@@ -166,7 +166,7 @@ def fast_parent_copy(obj):
setattr(new_obj, key, new_elements[value])
except KeyError:
pass
elif key in ['parent_stmt', 'parent_function']:
elif key in ['parent_stmt', 'parent_function', 'set_parent']:
continue
elif isinstance(value, list):
setattr(new_obj, key, list_rec(value))

86
jedi/parsing.py
View File

@@ -67,6 +67,8 @@ class Simple(Base):
self.start_pos = start_pos
self.end_pos = end_pos
self.parent = None
# use this attribute if parent should be something else than self.
self.set_parent = self
@Python3Method
def get_parent_until(self, classes=(), reverse=False,
@@ -111,12 +113,12 @@ class Scope(Simple):
self.asserts = []
def add_scope(self, sub, decorators):
sub.parent = self
sub.parent = self.set_parent
sub.decorators = decorators
for d in decorators:
# the parent is the same, because the decorator has not the scope
# of the function
d.parent = sub.parent
d.parent = self.set_parent
self.subscopes.append(sub)
return sub
@@ -125,7 +127,7 @@ class Scope(Simple):
Used to add a Statement or a Scope.
A statement would be a normal command (Statement) or a Scope (Flow).
"""
stmt.parent = self
stmt.parent = self.set_parent
self.statements.append(stmt)
return stmt
@@ -135,7 +137,7 @@ class Scope(Simple):
def add_import(self, imp):
self.imports.append(imp)
imp.parent = self
imp.parent = self.set_parent
def get_imports(self):
""" Gets also the imports within flow statements """
@@ -245,7 +247,7 @@ class SubModule(Scope, Module):
Depending on the underlying parser this may be a full module or just a part
of a module.
"""
def __init__(self, path, start_pos):
def __init__(self, path, start_pos, top_module=None):
super(SubModule, self).__init__(start_pos)
self.path = path
self.global_vars = []
@@ -255,6 +257,8 @@ class SubModule(Scope, Module):
# this may be changed depending on fast_parser
self._line_offset = 0
self.set_parent = top_module or self
def add_global(self, name):
"""
Global means in these context a function (subscope) which has a global
@@ -285,7 +289,7 @@ class SubModule(Scope, Module):
self.path)
string = r.group(1)
names = [(string, (0, 0))]
self._name = Name(names, self.start_pos, self.end_pos, self)
self._name = Name(names, self.start_pos, self.end_pos, self.set_parent)
return self._name
def is_builtin(self):
@@ -308,10 +312,10 @@ class Class(Scope):
def __init__(self, name, supers, start_pos, docstr=''):
super(Class, self).__init__(start_pos, docstr)
self.name = name
name.parent = self
name.parent = self.set_parent
self.supers = supers
for s in self.supers:
s.parent = self
s.parent = self.set_parent
self.decorators = []
def get_code(self, first_indent=False, indention=' '):
@@ -343,18 +347,18 @@ class Function(Scope):
def __init__(self, name, params, start_pos, annotation):
Scope.__init__(self, start_pos)
self.name = name
name.parent = self
name.parent = self.set_parent
self.params = params
for p in params:
p.parent = self
p.parent_function = self
p.parent = self.set_parent
p.parent_function = self.set_parent
self.decorators = []
self.returns = []
self.is_generator = False
self.listeners = set() # not used here, but in evaluation.
if annotation is not None:
annotation.parent = self
annotation.parent = self.set_parent
self.annotation = annotation
def get_code(self, first_indent=False, indention=' '):
@@ -437,14 +441,14 @@ class Flow(Scope):
# These have to be statements, because of with, which takes multiple.
self.inits = inits
for s in inits:
s.parent = self
s.parent = self.set_parent
if set_vars is None:
self.set_vars = []
else:
self.set_vars = set_vars
for s in self.set_vars:
s.parent.parent = self
s.parent = self
s.parent.parent = self.set_parent
s.parent = self.set_parent
@property
def parent(self):
@@ -551,7 +555,7 @@ class Import(Simple):
self.from_ns = from_ns
for n in [namespace, alias, from_ns]:
if n:
n.parent = self
n.parent = self.set_parent
self.star = star
self.relative_count = relative_count
@@ -634,7 +638,7 @@ class Statement(Simple):
self.used_vars = used_vars
self.token_list = token_list
for s in set_vars + used_funcs + used_vars:
s.parent = self
s.parent = self.set_parent
self.set_vars = self._remove_executions_from_set_vars(set_vars)
# cache
@@ -854,7 +858,7 @@ class Param(Statement):
self.parent_function = None
def add_annotation(self, annotation_stmt):
annotation_stmt.parent = self
annotation_stmt.parent = self.set_parent
self.annotation_stmt = annotation_stmt
def get_name(self):
@@ -1148,16 +1152,18 @@ class PyFuzzyParser(object):
:type user_position: tuple(int, int)
:param no_docstr: If True, a string at the beginning is not a docstr.
:param stop_on_scope: Stop if a scope appears -> for fast_parser
:param top_module: Use this module as a parent instead of `self.module`.
"""
def __init__(self, code, module_path=None, user_position=None,
no_docstr=False, line_offset=0, stop_on_scope=None):
no_docstr=False, line_offset=0, stop_on_scope=None,
top_module=None):
self.user_position = user_position
self.user_scope = None
self.user_stmt = None
self.no_docstr = no_docstr
# initialize global Scope
self.module = SubModule(module_path, (line_offset + 1, 0))
self.module = SubModule(module_path, (line_offset + 1, 0), top_module)
self.scope = self.module
self.current = (None, None)
self.start_pos = 1, 0
@@ -1171,12 +1177,20 @@ class PyFuzzyParser(object):
buf = StringIO(code)
self.gen = common.NoErrorTokenizer(buf.readline, line_offset,
stop_on_scope)
self.top_module = top_module or self.module
self.parse()
# clean up unused decorators
for d in self._decorators:
# set a parent for unused decorators, avoid NullPointerException
# because of `self.module.used_names`.
d.parent = self.module
def __repr__(self):
return "<%s: %s>" % (type(self).__name__, self.module)
def _check_user_stmt(self, simple):
# this is not user checking, just update the used_names
if not isinstance(simple, Param):
for tok_name in self.module.temp_used_names:
try:
@@ -1488,7 +1502,7 @@ class PyFuzzyParser(object):
in_clause.parent = self.scope
in_clause.parent = self.scope
debug.warning('list comprehension in_clause %s@%s'
% (tok, self.start_pos[0]))
% (repr(tok), self.start_pos[0]))
continue
other_level = 0
@@ -1620,7 +1634,7 @@ class PyFuzzyParser(object):
extended_flow = ['else', 'elif', 'except', 'finally']
statement_toks = ['{', '[', '(', '`']
decorators = []
self._decorators = []
self.freshscope = True
self.iterator = iter(self)
# This iterator stuff is not intentional. It grew historically.
@@ -1634,6 +1648,9 @@ class PyFuzzyParser(object):
if self.start_pos[1] <= self.scope.start_pos[1]:
self.scope.end_pos = self.start_pos
self.scope = self.scope.parent
if isinstance(self.scope, Module) \
and not isinstance(self.scope, SubModule):
self.scope = self.module
# check again for unindented stuff. this is true for syntax
# errors. only check for names, because thats relevant here. If
@@ -1643,7 +1660,12 @@ class PyFuzzyParser(object):
and self.scope != self.module:
self.scope.end_pos = self.start_pos
self.scope = self.scope.parent
if isinstance(self.scope, Module) \
and not isinstance(self.scope, SubModule):
self.scope = self.module
set_parent_scope = self.top_module if isinstance(self.scope,
SubModule) else self.scope
first_pos = self.start_pos
if tok == 'def':
func = self._parsefunction()
@@ -1652,16 +1674,16 @@ class PyFuzzyParser(object):
self.start_pos[0])
continue
self.freshscope = True
self.scope = self.scope.add_scope(func, decorators)
decorators = []
self.scope = self.scope.add_scope(func, self._decorators)
self._decorators = []
elif tok == 'class':
cls = self._parseclass()
if cls is None:
debug.warning("class: syntax error@%s" % self.start_pos[0])
continue
self.freshscope = True
self.scope = self.scope.add_scope(cls, decorators)
decorators = []
self.scope = self.scope.add_scope(cls, self._decorators)
self._decorators = []
# import stuff
elif tok == 'import':
imports = self._parseimportlist()
@@ -1717,14 +1739,14 @@ class PyFuzzyParser(object):
debug.warning('syntax err, for flow started @%s',
self.start_pos[0])
if statement is not None:
statement.parent = self.scope
statement.parent = set_parent_scope
if set_stmt is not None:
set_stmt.parent = self.scope
set_stmt.parent = set_parent_scope
else:
debug.warning('syntax err, for flow incomplete @%s',
self.start_pos[0])
if set_stmt is not None:
set_stmt.parent = self.scope
set_stmt.parent = set_parent_scope
elif tok in ['if', 'while', 'try', 'with'] + extended_flow:
added_breaks = []
@@ -1765,7 +1787,7 @@ class PyFuzzyParser(object):
self.scope = s
else:
for i in inits:
i.parent = self.scope
i.parent = set_parent_scope
debug.warning('syntax err, flow started @%s',
self.start_pos[0])
# globals
@@ -1780,12 +1802,12 @@ class PyFuzzyParser(object):
# decorator
elif tok == '@':
stmt, tok = self._parse_statement()
decorators.append(stmt)
self._decorators.append(stmt)
elif tok == 'pass':
continue
elif tok == 'assert':
stmt, tok = self._parse_statement()
stmt.parent = self.scope
stmt.parent = set_parent_scope
self.scope.asserts.append(stmt)
# default
elif token_type in [tokenize.NAME, tokenize.STRING,

2
jedi/settings.py
View File

@@ -19,7 +19,7 @@ add_bracket_after_function = False
# ----------------
# Use the fast parser, may cause problems sometimes.
fast_parser = False
fast_parser = True
# ----------------
# dynamic stuff

4
test/run.py
View File

@@ -127,8 +127,8 @@ def run_test(source, f_name, lines_to_execute):
try:
should_be |= defs(line_nr - 1, start + correct_start)
except Exception:
raise Exception('could not resolve %s indent %s'
% (line_nr - 1, start))
print('could not resolve %s indent %s' % (line_nr - 1, start))
raise
if print_debug:
api.set_debug_function(debug.print_to_stdout)
# because the objects have different ids, `repr` it, then compare it.