VimPlug/jedi
1
0
Fork 1
mirror of https://github.com/davidhalter/jedi.git synced 2025-12-07 22:44:27 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

many bugfixes related to fast_parser

Browse Source
This commit is contained in:
David Halter
2012-12-15 16:38:21 +01:00
parent b8b4a02398
commit 2b579eb09b
8 changed files with 95 additions and 52 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
jedi/builtin.py
View File

@@ -9,9 +9,11 @@ if is_py3k:
import types import types
import inspect import inspect
import settings
import common import common
import debug import debug
import parsing import parsing
import fast_parser
import imports import imports
import evaluate import evaluate
@@ -67,7 +69,11 @@ class CachedModule(object):
def _load_module(self): def _load_module(self):
source = self._get_source() source = self._get_source()
self._parser = parsing.PyFuzzyParser(source, self.path or self.name) p = self.path or self.name
if settings.fast_parser:
self._parser = fast_parser.FastParser(source, p)
else:
self._parser = parsing.PyFuzzyParser(source, p)
p_time = None if not self.path else os.path.getmtime(self.path) p_time = None if not self.path else os.path.getmtime(self.path)
if self.path or self.name: if self.path or self.name:

2
jedi/common.py
View File

@@ -24,7 +24,7 @@ class MultiLevelAttributeError(Exception):
:param base: return values of sys.exc_info(). :param base: return values of sys.exc_info().
""" """
def __init__(self, base): def __init__(self, base=None):
self.base = base self.base = base
def __str__(self): def __str__(self):

11
jedi/evaluate.py
View File

@@ -889,10 +889,10 @@ def get_names_for_scope(scope, position=None, star_search=True,
# Ignore the Flows, because the classes and functions care for that. # Ignore the Flows, because the classes and functions care for that.
# InstanceElement of Class is ignored, if it is not the start scope. # InstanceElement of Class is ignored, if it is not the start scope.
if not (scope != non_flow and scope.isinstance(parsing.Class) if not (scope != non_flow and scope.isinstance(parsing.Class)
or scope.isinstance(parsing.Flow) or scope.isinstance(parsing.Flow)
or scope.isinstance(Instance) and non_flow.isinstance(Function) or scope.isinstance(Instance)
or isinstance(scope, parsing.SubModule) and scope.parent and non_flow.isinstance(Function)
): ):
try: try:
if isinstance(scope, Instance): if isinstance(scope, Instance):
for g in scope.scope_generator(): for g in scope.scope_generator():
@@ -1100,9 +1100,6 @@ def get_scopes_for_name(scope, name_str, position=None, search_global=False,
if isinstance(p, InstanceElement) \ if isinstance(p, InstanceElement) \
and isinstance(p.var, parsing.Class): and isinstance(p.var, parsing.Class):
p = p.var p = p.var
if isinstance(p, parsing.SubModule) and p.parent is not None:
p = p.parent
if name_str == name.get_code() and p not in break_scopes: if name_str == name.get_code() and p not in break_scopes:
r, no_break_scope = process(name) r, no_break_scope = process(name)
if is_goto: if is_goto:

32
jedi/fast_parser.py
View File

@@ -13,6 +13,7 @@ class Module(parsing.Simple, parsing.Module):
def __init__(self, parsers): def __init__(self, parsers):
super(Module, self).__init__((1,0)) super(Module, self).__init__((1,0))
self.parsers = parsers self.parsers = parsers
self._end_pos = None, None
self.reset_caches() self.reset_caches()
def reset_caches(self): def reset_caches(self):
@@ -47,7 +48,7 @@ class Module(parsing.Simple, parsing.Module):
elif name in properties: elif name in properties:
return self._get(name, properties[name]) return self._get(name, properties[name])
else: else:
raise AttributeError() raise AttributeError("__getattr__ doesn't offer %s" % name)
def get_statement_for_position(self, pos): def get_statement_for_position(self, pos):
key = 'get_statement_for_position', pos key = 'get_statement_for_position', pos
@@ -63,6 +64,8 @@ class Module(parsing.Simple, parsing.Module):
@property @property
def used_names(self): def used_names(self):
if not self.parsers:
raise NotImplementedError("Parser doesn't exist.")
key = 'used_names' key = 'used_names'
if key not in self.cache: if key not in self.cache:
dct = {} dct = {}
@@ -78,27 +81,36 @@ class Module(parsing.Simple, parsing.Module):
@property @property
def docstr(self): def docstr(self):
if not self.parsers:
raise NotImplementedError("Parser doesn't exist.")
return self.parsers[0].module.docstr return self.parsers[0].module.docstr
@property @property
def name(self): def name(self):
if not self.parsers:
raise NotImplementedError("Parser doesn't exist.")
return self.parsers[0].module.name return self.parsers[0].module.name
@property @property
def path(self): def path(self):
if not self.parsers:
raise NotImplementedError("Parser doesn't exist.")
return self.parsers[0].module.path return self.parsers[0].module.path
@property @property
def is_builtin(self): def is_builtin(self):
if not self.parsers:
raise NotImplementedError("Parser doesn't exist.")
return self.parsers[0].module.is_builtin return self.parsers[0].module.is_builtin
@property @property
def end_pos(self): def end_pos(self):
return self.parsers[-1].module.end_pos return self._end_pos
@end_pos.setter @end_pos.setter
def end_pos(self, value): def end_pos(self, value):
pass # just ignore, end_pos is not important if None not in value and self._end_pos < value:
self._end_pos = value
def __repr__(self): def __repr__(self):
return "<%s: %s@%s-%s>" % (type(self).__name__, self.name, return "<%s: %s@%s-%s>" % (type(self).__name__, self.name,
@@ -153,17 +165,23 @@ class FastParser(use_metaclass(CachedFastParser)):
def update(self, code, user_position=None): def update(self, code, user_position=None):
self.user_position = user_position self.user_position = user_position
self.reset_caches() self.reset_caches()
self.parsers = [] # TODO remove self.parsers = []
self.module.parsers = self.parsers
self._parse(code) self._parse(code)
def _parse(self, code): def _parse(self, code):
parts = re.findall(r'(?:\n(?:def|class)|^).*?(?=\n(?:def|class)|$)', """ :type code: str """
code, re.DOTALL) r = r'(?:\n(?:def|class|@.*?\n(?:def|class))|^).*?(?=\n(?:def|class|@)|$)'
parts = re.findall(r, code, re.DOTALL)
line_offset = 0 line_offset = 0
for p in parts: for p in parts:
lines = p.count('\n') lines = p.count('\n')
p = parsing.PyFuzzyParser(p, self.module_path, self.user_position, p = parsing.PyFuzzyParser(p, self.module_path, self.user_position,
line_offset=line_offset, stop_on_scope=True) line_offset=line_offset, stop_on_scope=True,
top_module=self.module)
p.module.parent = self.module p.module.parent = self.module
line_offset += lines line_offset += lines
self.parsers.append(p) self.parsers.append(p)

2
jedi/helpers.py
View File

@@ -166,7 +166,7 @@ def fast_parent_copy(obj):
setattr(new_obj, key, new_elements[value]) setattr(new_obj, key, new_elements[value])
except KeyError: except KeyError:
pass pass
elif key in ['parent_stmt', 'parent_function']: elif key in ['parent_stmt', 'parent_function', 'set_parent']:
continue continue
elif isinstance(value, list): elif isinstance(value, list):
setattr(new_obj, key, list_rec(value)) setattr(new_obj, key, list_rec(value))

86
jedi/parsing.py
View File

@@ -67,6 +67,8 @@ class Simple(Base):
self.start_pos = start_pos self.start_pos = start_pos
self.end_pos = end_pos self.end_pos = end_pos
self.parent = None self.parent = None
# use this attribute if parent should be something else than self.
self.set_parent = self
@Python3Method @Python3Method
def get_parent_until(self, classes=(), reverse=False, def get_parent_until(self, classes=(), reverse=False,
@@ -111,12 +113,12 @@ class Scope(Simple):
self.asserts = [] self.asserts = []
def add_scope(self, sub, decorators): def add_scope(self, sub, decorators):
sub.parent = self sub.parent = self.set_parent
sub.decorators = decorators sub.decorators = decorators
for d in decorators: for d in decorators:
# the parent is the same, because the decorator has not the scope # the parent is the same, because the decorator has not the scope
# of the function # of the function
d.parent = sub.parent d.parent = self.set_parent
self.subscopes.append(sub) self.subscopes.append(sub)
return sub return sub
@@ -125,7 +127,7 @@ class Scope(Simple):
Used to add a Statement or a Scope. Used to add a Statement or a Scope.
A statement would be a normal command (Statement) or a Scope (Flow). A statement would be a normal command (Statement) or a Scope (Flow).
""" """
stmt.parent = self stmt.parent = self.set_parent
self.statements.append(stmt) self.statements.append(stmt)
return stmt return stmt
@@ -135,7 +137,7 @@ class Scope(Simple):
def add_import(self, imp): def add_import(self, imp):
self.imports.append(imp) self.imports.append(imp)
imp.parent = self imp.parent = self.set_parent
def get_imports(self): def get_imports(self):
""" Gets also the imports within flow statements """ """ Gets also the imports within flow statements """
@@ -245,7 +247,7 @@ class SubModule(Scope, Module):
Depending on the underlying parser this may be a full module or just a part Depending on the underlying parser this may be a full module or just a part
of a module. of a module.
""" """
def __init__(self, path, start_pos): def __init__(self, path, start_pos, top_module=None):
super(SubModule, self).__init__(start_pos) super(SubModule, self).__init__(start_pos)
self.path = path self.path = path
self.global_vars = [] self.global_vars = []
@@ -255,6 +257,8 @@ class SubModule(Scope, Module):
# this may be changed depending on fast_parser # this may be changed depending on fast_parser
self._line_offset = 0 self._line_offset = 0
self.set_parent = top_module or self
def add_global(self, name): def add_global(self, name):
""" """
Global means in these context a function (subscope) which has a global Global means in these context a function (subscope) which has a global
@@ -285,7 +289,7 @@ class SubModule(Scope, Module):
self.path) self.path)
string = r.group(1) string = r.group(1)
names = [(string, (0, 0))] names = [(string, (0, 0))]
self._name = Name(names, self.start_pos, self.end_pos, self) self._name = Name(names, self.start_pos, self.end_pos, self.set_parent)
return self._name return self._name
def is_builtin(self): def is_builtin(self):
@@ -308,10 +312,10 @@ class Class(Scope):
def __init__(self, name, supers, start_pos, docstr=''): def __init__(self, name, supers, start_pos, docstr=''):
super(Class, self).__init__(start_pos, docstr) super(Class, self).__init__(start_pos, docstr)
self.name = name self.name = name
name.parent = self name.parent = self.set_parent
self.supers = supers self.supers = supers
for s in self.supers: for s in self.supers:
s.parent = self s.parent = self.set_parent
self.decorators = [] self.decorators = []
def get_code(self, first_indent=False, indention=' '): def get_code(self, first_indent=False, indention=' '):
@@ -343,18 +347,18 @@ class Function(Scope):
def __init__(self, name, params, start_pos, annotation): def __init__(self, name, params, start_pos, annotation):
Scope.__init__(self, start_pos) Scope.__init__(self, start_pos)
self.name = name self.name = name
name.parent = self name.parent = self.set_parent
self.params = params self.params = params
for p in params: for p in params:
p.parent = self p.parent = self.set_parent
p.parent_function = self p.parent_function = self.set_parent
self.decorators = [] self.decorators = []
self.returns = [] self.returns = []
self.is_generator = False self.is_generator = False
self.listeners = set() # not used here, but in evaluation. self.listeners = set() # not used here, but in evaluation.
if annotation is not None: if annotation is not None:
annotation.parent = self annotation.parent = self.set_parent
self.annotation = annotation self.annotation = annotation
def get_code(self, first_indent=False, indention=' '): def get_code(self, first_indent=False, indention=' '):
@@ -437,14 +441,14 @@ class Flow(Scope):
# These have to be statements, because of with, which takes multiple. # These have to be statements, because of with, which takes multiple.
self.inits = inits self.inits = inits
for s in inits: for s in inits:
s.parent = self s.parent = self.set_parent
if set_vars is None: if set_vars is None:
self.set_vars = [] self.set_vars = []
else: else:
self.set_vars = set_vars self.set_vars = set_vars
for s in self.set_vars: for s in self.set_vars:
s.parent.parent = self s.parent.parent = self.set_parent
s.parent = self s.parent = self.set_parent
@property @property
def parent(self): def parent(self):
@@ -551,7 +555,7 @@ class Import(Simple):
self.from_ns = from_ns self.from_ns = from_ns
for n in [namespace, alias, from_ns]: for n in [namespace, alias, from_ns]:
if n: if n:
n.parent = self n.parent = self.set_parent
self.star = star self.star = star
self.relative_count = relative_count self.relative_count = relative_count
@@ -634,7 +638,7 @@ class Statement(Simple):
self.used_vars = used_vars self.used_vars = used_vars
self.token_list = token_list self.token_list = token_list
for s in set_vars + used_funcs + used_vars: for s in set_vars + used_funcs + used_vars:
s.parent = self s.parent = self.set_parent
self.set_vars = self._remove_executions_from_set_vars(set_vars) self.set_vars = self._remove_executions_from_set_vars(set_vars)
# cache # cache
@@ -854,7 +858,7 @@ class Param(Statement):
self.parent_function = None self.parent_function = None
def add_annotation(self, annotation_stmt): def add_annotation(self, annotation_stmt):
annotation_stmt.parent = self annotation_stmt.parent = self.set_parent
self.annotation_stmt = annotation_stmt self.annotation_stmt = annotation_stmt
def get_name(self): def get_name(self):
@@ -1148,16 +1152,18 @@ class PyFuzzyParser(object):
:type user_position: tuple(int, int) :type user_position: tuple(int, int)
:param no_docstr: If True, a string at the beginning is not a docstr. :param no_docstr: If True, a string at the beginning is not a docstr.
:param stop_on_scope: Stop if a scope appears -> for fast_parser :param stop_on_scope: Stop if a scope appears -> for fast_parser
:param top_module: Use this module as a parent instead of `self.module`.
""" """
def __init__(self, code, module_path=None, user_position=None, def __init__(self, code, module_path=None, user_position=None,
no_docstr=False, line_offset=0, stop_on_scope=None): no_docstr=False, line_offset=0, stop_on_scope=None,
top_module=None):
self.user_position = user_position self.user_position = user_position
self.user_scope = None self.user_scope = None
self.user_stmt = None self.user_stmt = None
self.no_docstr = no_docstr self.no_docstr = no_docstr
# initialize global Scope # initialize global Scope
self.module = SubModule(module_path, (line_offset + 1, 0)) self.module = SubModule(module_path, (line_offset + 1, 0), top_module)
self.scope = self.module self.scope = self.module
self.current = (None, None) self.current = (None, None)
self.start_pos = 1, 0 self.start_pos = 1, 0
@@ -1171,12 +1177,20 @@ class PyFuzzyParser(object):
buf = StringIO(code) buf = StringIO(code)
self.gen = common.NoErrorTokenizer(buf.readline, line_offset, self.gen = common.NoErrorTokenizer(buf.readline, line_offset,
stop_on_scope) stop_on_scope)
self.top_module = top_module or self.module
self.parse() self.parse()
# clean up unused decorators
for d in self._decorators:
# set a parent for unused decorators, avoid NullPointerException
# because of `self.module.used_names`.
d.parent = self.module
def __repr__(self): def __repr__(self):
return "<%s: %s>" % (type(self).__name__, self.module) return "<%s: %s>" % (type(self).__name__, self.module)
def _check_user_stmt(self, simple): def _check_user_stmt(self, simple):
# this is not user checking, just update the used_names
if not isinstance(simple, Param): if not isinstance(simple, Param):
for tok_name in self.module.temp_used_names: for tok_name in self.module.temp_used_names:
try: try:
@@ -1488,7 +1502,7 @@ class PyFuzzyParser(object):
in_clause.parent = self.scope in_clause.parent = self.scope
in_clause.parent = self.scope in_clause.parent = self.scope
debug.warning('list comprehension in_clause %s@%s' debug.warning('list comprehension in_clause %s@%s'
% (tok, self.start_pos[0])) % (repr(tok), self.start_pos[0]))
continue continue
other_level = 0 other_level = 0
@@ -1620,7 +1634,7 @@ class PyFuzzyParser(object):
extended_flow = ['else', 'elif', 'except', 'finally'] extended_flow = ['else', 'elif', 'except', 'finally']
statement_toks = ['{', '[', '(', '`'] statement_toks = ['{', '[', '(', '`']
decorators = [] self._decorators = []
self.freshscope = True self.freshscope = True
self.iterator = iter(self) self.iterator = iter(self)
# This iterator stuff is not intentional. It grew historically. # This iterator stuff is not intentional. It grew historically.
@@ -1634,6 +1648,9 @@ class PyFuzzyParser(object):
if self.start_pos[1] <= self.scope.start_pos[1]: if self.start_pos[1] <= self.scope.start_pos[1]:
self.scope.end_pos = self.start_pos self.scope.end_pos = self.start_pos
self.scope = self.scope.parent self.scope = self.scope.parent
if isinstance(self.scope, Module) \
and not isinstance(self.scope, SubModule):
self.scope = self.module
# check again for unindented stuff. this is true for syntax # check again for unindented stuff. this is true for syntax
# errors. only check for names, because thats relevant here. If # errors. only check for names, because thats relevant here. If
@@ -1643,7 +1660,12 @@ class PyFuzzyParser(object):
and self.scope != self.module: and self.scope != self.module:
self.scope.end_pos = self.start_pos self.scope.end_pos = self.start_pos
self.scope = self.scope.parent self.scope = self.scope.parent
if isinstance(self.scope, Module) \
and not isinstance(self.scope, SubModule):
self.scope = self.module
set_parent_scope = self.top_module if isinstance(self.scope,
SubModule) else self.scope
first_pos = self.start_pos first_pos = self.start_pos
if tok == 'def': if tok == 'def':
func = self._parsefunction() func = self._parsefunction()
@@ -1652,16 +1674,16 @@ class PyFuzzyParser(object):
self.start_pos[0]) self.start_pos[0])
continue continue
self.freshscope = True self.freshscope = True
self.scope = self.scope.add_scope(func, decorators) self.scope = self.scope.add_scope(func, self._decorators)
decorators = [] self._decorators = []
elif tok == 'class': elif tok == 'class':
cls = self._parseclass() cls = self._parseclass()
if cls is None: if cls is None:
debug.warning("class: syntax error@%s" % self.start_pos[0]) debug.warning("class: syntax error@%s" % self.start_pos[0])
continue continue
self.freshscope = True self.freshscope = True
self.scope = self.scope.add_scope(cls, decorators) self.scope = self.scope.add_scope(cls, self._decorators)
decorators = [] self._decorators = []
# import stuff # import stuff
elif tok == 'import': elif tok == 'import':
imports = self._parseimportlist() imports = self._parseimportlist()
@@ -1717,14 +1739,14 @@ class PyFuzzyParser(object):
debug.warning('syntax err, for flow started @%s', debug.warning('syntax err, for flow started @%s',
self.start_pos[0]) self.start_pos[0])
if statement is not None: if statement is not None:
statement.parent = self.scope statement.parent = set_parent_scope
if set_stmt is not None: if set_stmt is not None:
set_stmt.parent = self.scope set_stmt.parent = set_parent_scope
else: else:
debug.warning('syntax err, for flow incomplete @%s', debug.warning('syntax err, for flow incomplete @%s',
self.start_pos[0]) self.start_pos[0])
if set_stmt is not None: if set_stmt is not None:
set_stmt.parent = self.scope set_stmt.parent = set_parent_scope
elif tok in ['if', 'while', 'try', 'with'] + extended_flow: elif tok in ['if', 'while', 'try', 'with'] + extended_flow:
added_breaks = [] added_breaks = []
@@ -1765,7 +1787,7 @@ class PyFuzzyParser(object):
self.scope = s self.scope = s
else: else:
for i in inits: for i in inits:
i.parent = self.scope i.parent = set_parent_scope
debug.warning('syntax err, flow started @%s', debug.warning('syntax err, flow started @%s',
self.start_pos[0]) self.start_pos[0])
# globals # globals
@@ -1780,12 +1802,12 @@ class PyFuzzyParser(object):
# decorator # decorator
elif tok == '@': elif tok == '@':
stmt, tok = self._parse_statement() stmt, tok = self._parse_statement()
decorators.append(stmt) self._decorators.append(stmt)
elif tok == 'pass': elif tok == 'pass':
continue continue
elif tok == 'assert': elif tok == 'assert':
stmt, tok = self._parse_statement() stmt, tok = self._parse_statement()
stmt.parent = self.scope stmt.parent = set_parent_scope
self.scope.asserts.append(stmt) self.scope.asserts.append(stmt)
# default # default
elif token_type in [tokenize.NAME, tokenize.STRING, elif token_type in [tokenize.NAME, tokenize.STRING,

2
jedi/settings.py
View File

@@ -19,7 +19,7 @@ add_bracket_after_function = False
# ---------------- # ----------------
# Use the fast parser, may cause problems sometimes. # Use the fast parser, may cause problems sometimes.
fast_parser = False fast_parser = True
# ---------------- # ----------------
# dynamic stuff # dynamic stuff

4
test/run.py
View File

@@ -127,8 +127,8 @@ def run_test(source, f_name, lines_to_execute):
try: try:
should_be |= defs(line_nr - 1, start + correct_start) should_be |= defs(line_nr - 1, start + correct_start)
except Exception: except Exception:
raise Exception('could not resolve %s indent %s' print('could not resolve %s indent %s' % (line_nr - 1, start))
% (line_nr - 1, start)) raise
if print_debug: if print_debug:
api.set_debug_function(debug.print_to_stdout) api.set_debug_function(debug.print_to_stdout)
# because the objects have different ids, `repr` it, then compare it. # because the objects have different ids, `repr` it, then compare it.