VimPlug/jedi
1
0
Fork 1
mirror of https://github.com/davidhalter/jedi.git synced 2025-12-09 23:34:45 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

line_nr/indent were now completely replaced by start_pos. tests are failing!

Browse Source
This commit is contained in:
David Halter
2012-07-14 03:06:49 +02:00
parent cdf97b2bbe
commit 08a4562958
5 changed files with 174 additions and 190 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
evaluate.py
View File

@@ -211,7 +211,7 @@ class Instance(Executable):
if name == 'get_index_types': if name == 'get_index_types':
# TODO Call __getitem__ in such cases? # TODO Call __getitem__ in such cases?
return lambda: [] return lambda: []
if name not in ['line_nr', 'indent', 'name', 'get_imports']: if name not in ['start_pos', 'end_pos', 'name', 'get_imports']:
raise AttributeError("Instance %s: Don't touch this (%s)!" raise AttributeError("Instance %s: Don't touch this (%s)!"
% (self, name)) % (self, name))
return getattr(self.base, name) return getattr(self.base, name)
@@ -303,7 +303,7 @@ class Class(object):
return self.base.name return self.base.name
def __getattr__(self, name): def __getattr__(self, name):
if name not in ['line_nr', 'indent', 'parent', 'subscopes', if name not in ['start_pos', 'end_pos', 'parent', 'subscopes',
'get_imports']: 'get_imports']:
raise AttributeError("Don't touch this (%s)!" % name) raise AttributeError("Don't touch this (%s)!" % name)
return getattr(self.base, name) return getattr(self.base, name)
@@ -611,7 +611,7 @@ class Execution(Executable):
return objects return objects
def __getattr__(self, name): def __getattr__(self, name):
if name not in ['indent', 'line_nr', 'imports']: if name not in ['start_pos', 'end_pos', 'imports']:
raise AttributeError('Tried to access %s: %s. Why?' % (name, self)) raise AttributeError('Tried to access %s: %s. Why?' % (name, self))
return getattr(self.base, name) return getattr(self.base, name)
@@ -765,7 +765,7 @@ class ArrayElement(object):
def __getattr__(self, name): def __getattr__(self, name):
# Set access privileges: # Set access privileges:
if name not in ['parent', 'names', 'line_nr', 'indent']: if name not in ['parent', 'names', 'start_pos', 'end_pos']:
raise AttributeError('Strange access: %s.' % name) raise AttributeError('Strange access: %s.' % name)
return getattr(self.name, name) return getattr(self.name, name)
@@ -775,7 +775,7 @@ class ArrayElement(object):
def get_defined_names_for_position(obj, position=None, start_scope=None): def get_defined_names_for_position(obj, position=None, start_scope=None):
""" """
:param position: the position as a row/column tuple, default is infinity. :param position: the position as a line/column tuple, default is infinity.
""" """
names = obj.get_defined_names() names = obj.get_defined_names()
# Instances have special rules, always return all the possible completions, # Instances have special rules, always return all the possible completions,
@@ -787,7 +787,7 @@ def get_defined_names_for_position(obj, position=None, start_scope=None):
return names return names
names_new = [] names_new = []
for n in names: for n in names:
if (n.line_nr, n.indent) < position: if (n.start_pos) < position:
names_new.append(n) names_new.append(n)
return names_new return names_new
@@ -826,7 +826,7 @@ def get_names_for_scope(scope, position=None, star_search=True):
def get_scopes_for_name(scope, name_str, position=None, search_global=False): def get_scopes_for_name(scope, name_str, position=None, search_global=False):
""" """
:param position: Position of the last statement ->tuple of line, indent :param position: Position of the last statement -> tuple of line, column
:return: List of Names. Their parents are the scopes, they are defined in. :return: List of Names. Their parents are the scopes, they are defined in.
:rtype: list :rtype: list
""" """
@@ -902,8 +902,8 @@ def get_scopes_for_name(scope, name_str, position=None, search_global=False):
return result return result
result = [] result = []
# compare func uses the tuple of line/indent = row/column # compare func uses the tuple of line/indent = line/column
comparison_func = lambda name: (name.line_nr, name.indent) comparison_func = lambda name: (name.start_pos)
for scope, name_list in scope_generator: for scope, name_list in scope_generator:
break_scopes = [] break_scopes = []
# here is the position stuff happening (sorting of variables) # here is the position stuff happening (sorting of variables)
@@ -1080,7 +1080,7 @@ def follow_call(call):
scope = call.parent_stmt.parent scope = call.parent_stmt.parent
path = call.generate_call_list() path = call.generate_call_list()
position = (call.parent_stmt.line_nr, call.parent_stmt.indent) position = call.parent_stmt.start_pos
current = next(path) current = next(path)
if isinstance(current, parsing.Array): if isinstance(current, parsing.Array):

17
functions.py
View File

@@ -121,14 +121,14 @@ def get_completion_parts(path):
return match.groups() return match.groups()
def complete(source, row, column, source_path): def complete(source, line, column, source_path):
""" """
An auto completer for python files. An auto completer for python files.
:param source: The source code of the current file :param source: The source code of the current file
:type source: string :type source: string
:param row: The row to complete in. :param line: The line to complete in.
:type row: int :type line: int
:param col: The column to complete in. :param col: The column to complete in.
:type col: int :type col: int
:param source_path: The path in the os, the current module is in. :param source_path: The path in the os, the current module is in.
@@ -138,7 +138,7 @@ def complete(source, row, column, source_path):
:rtype: list :rtype: list
""" """
try: try:
scopes, path, dot, like = prepare_goto(source, (row, column), scopes, path, dot, like = prepare_goto(source, (line, column),
source_path, True) source_path, True)
except NotFoundError: except NotFoundError:
# normally this would be used like this: `NotFoundError as exc`, but # normally this would be used like this: `NotFoundError as exc`, but
@@ -174,7 +174,6 @@ def complete(source, row, column, source_path):
def prepare_goto(source, position, source_path, is_like_search): def prepare_goto(source, position, source_path, is_like_search):
f = modules.ModuleWithCursor(source_path, source=source, position=position) f = modules.ModuleWithCursor(source_path, source=source, position=position)
scope = f.parser.user_scope scope = f.parser.user_scope
print scope
if is_like_search: if is_like_search:
path = f.get_path_until_cursor() path = f.get_path_until_cursor()
@@ -206,7 +205,7 @@ def prepare_goto(source, position, source_path, is_like_search):
return scopes return scopes
def get_definitions(source, row, column, source_path): def get_definitions(source, line, column, source_path):
""" """
Returns the definitions of a the path under the cursor. Returns the definitions of a the path under the cursor.
This is not a goto function! This follows complicated paths and returns the This is not a goto function! This follows complicated paths and returns the
@@ -214,8 +213,8 @@ def get_definitions(source, row, column, source_path):
:param source: The source code of the current file :param source: The source code of the current file
:type source: string :type source: string
:param row: The row to complete in. :param line: The line to complete in.
:type row: int :type line: int
:param col: The column to complete in. :param col: The column to complete in.
:type col: int :type col: int
:param source_path: The path in the os, the current module is in. :param source_path: The path in the os, the current module is in.
@@ -224,7 +223,7 @@ def get_definitions(source, row, column, source_path):
:return: list of Definition objects, which are basically scopes. :return: list of Definition objects, which are basically scopes.
:rtype: list :rtype: list
""" """
scopes = prepare_goto(source, (row, column), source_path, False) scopes = prepare_goto(source, (line, column), source_path, False)
_clear_caches() _clear_caches()
return [Definition(s) for s in set(scopes)] return [Definition(s) for s in set(scopes)]

2
helpers.py
View File

@@ -39,7 +39,7 @@ class RecursionDecorator(object):
class RecursionNode(object): class RecursionNode(object):
def __init__(self, stmt, parent): def __init__(self, stmt, parent):
self.script = stmt.get_parent_until() self.script = stmt.get_parent_until()
self.position = (stmt.line_nr, stmt.indent) self.position = stmt.start_pos
self.parent = parent self.parent = parent
def __eq__(self, other): def __eq__(self, other):

16
modules.py
View File

@@ -50,15 +50,15 @@ class ModuleWithCursor(Module):
:param source: The source code of the file. :param source: The source code of the file.
:param path: The module path of the file. :param path: The module path of the file.
:param row: The row, the user is currently in. Only important for the \ :param position: The position, the user is currently in. Only important \
main file. for the main file.
""" """
def __init__(self, path, source, position): def __init__(self, path, source, position):
super(ModuleWithCursor, self).__init__(path, source) super(ModuleWithCursor, self).__init__(path, source)
self.position = position self.position = position
# this two are only used, because there is no nonlocal in Python 2 # this two are only used, because there is no nonlocal in Python 2
self._row_temp = None self._line_temp = None
self._relevant_temp = None self._relevant_temp = None
# Call the parser already here, because it will be used anyways. # Call the parser already here, because it will be used anyways.
@@ -71,23 +71,23 @@ class ModuleWithCursor(Module):
self._is_first = True self._is_first = True
def fetch_line(): def fetch_line():
line = self.get_line(self._row_temp) line = self.get_line(self._line_temp)
if self._is_first: if self._is_first:
self._is_first = False self._is_first = False
line = line[:self.position[1]] line = line[:self.position[1]]
else: else:
line = line + '\n' line = line + '\n'
# add lines with a backslash at the end # add lines with a backslash at the end
while self._row_temp > 1: while self._line_temp > 1:
self._row_temp -= 1 self._line_temp -= 1
last_line = self.get_line(self._row_temp) last_line = self.get_line(self._line_temp)
if last_line and last_line[-1] == '\\': if last_line and last_line[-1] == '\\':
line = last_line[:-1] + ' ' + line line = last_line[:-1] + ' ' + line
else: else:
break break
return line[::-1] return line[::-1]
self._row_temp = self.position[0] self._line_temp = self.position[0]
force_point = False force_point = False
open_brackets = ['(', '[', '{'] open_brackets = ['(', '[', '{']

309
parsing.py
View File

@@ -56,10 +56,9 @@ class Simple(object):
The super class for Scope, Import, Name and Statement. Every object in The super class for Scope, Import, Name and Statement. Every object in
the parser tree inherits from this class. the parser tree inherits from this class.
""" """
def __init__(self, indent, line_nr, line_end=None): def __init__(self, start_pos, end_pos=(None, None)):
self.indent = indent self.start_pos = start_pos
self.line_nr = line_nr self.end_pos = end_pos
self.line_end = line_end
self.parent = None self.parent = None
def get_parent_until(self, *classes): def get_parent_until(self, *classes):
@@ -72,7 +71,7 @@ class Simple(object):
def __repr__(self): def __repr__(self):
code = self.get_code().replace('\n', ' ') code = self.get_code().replace('\n', ' ')
return "<%s: %s@%s>" % \ return "<%s: %s@%s>" % \
(self.__class__.__name__, code, self.line_nr) (self.__class__.__name__, code, self.start_pos[0])
class Scope(Simple): class Scope(Simple):
@@ -83,22 +82,20 @@ class Scope(Simple):
well as variables and imports. It is used to access the structure of python well as variables and imports. It is used to access the structure of python
files. files.
:param indent: The indent level of the flow statement. :param start_pos: The position (line and column) of the scope.
:type indent: int :type start_pos: tuple(int, int)
:param line_nr: Line number of the flow statement.
:type line_nr: int
:param docstr: The docstring for the current Scope. :param docstr: The docstring for the current Scope.
:type docstr: str :type docstr: str
""" """
def __init__(self, indent, line_nr, docstr=''): def __init__(self, start_pos, docstr=''):
super(Scope, self).__init__(indent, line_nr) super(Scope, self).__init__(start_pos)
self.subscopes = [] self.subscopes = []
self.imports = [] self.imports = []
self.statements = [] self.statements = []
self.docstr = docstr self.docstr = docstr
def add_scope(self, sub, decorators): def add_scope(self, sub, decorators):
# print 'push scope: [%s@%s]' % (sub.line_nr, sub.indent) # print 'push scope: [%s@%s]' % sub.start_pos
sub.parent = self sub.parent = self
sub.decorators = decorators sub.decorators = decorators
for d in decorators: for d in decorators:
@@ -164,7 +161,6 @@ class Scope(Simple):
for i in self.imports: for i in self.imports:
string += i.get_code() string += i.get_code()
for sub in self.subscopes: for sub in self.subscopes:
#string += str(sub.line_nr)
string += sub.get_code(first_indent=True, indention=indention) string += sub.get_code(first_indent=True, indention=indention)
for stmt in self.statements: for stmt in self.statements:
string += stmt.get_code() string += stmt.get_code()
@@ -225,8 +221,8 @@ class Scope(Simple):
except AttributeError: except AttributeError:
name = self.path name = self.path
return "<%s: %s@%s-%s>" % \ return "<%s: %s@%s-%s>" % (self.__class__.__name__, name,
(self.__class__.__name__, name, self.line_nr, self.line_end) self.start_pos[0], self.end_pos[0])
class Module(Scope): class Module(Scope):
@@ -234,7 +230,7 @@ class Module(Scope):
The top scope, which is always a module. The top scope, which is always a module.
""" """
def __init__(self, path, docstr=''): def __init__(self, path, docstr=''):
super(Module, self).__init__(0, 0, docstr) super(Module, self).__init__((0, 0), docstr)
self.path = path self.path = path
self.global_vars = [] self.global_vars = []
@@ -262,17 +258,15 @@ class Class(Scope):
:param name: The Class name. :param name: The Class name.
:type name: string :type name: string
:param name: The super classes of a Class. :param supers: The super classes of a Class.
:type name: list :type supers: list
:param indent: The indent level of the flow statement. :param start_pos: The start position (line, column) of the class.
:type indent: int :type start_pos: tuple(int, int)
:param line_nr: Line number of the flow statement.
:type line_nr: int
:param docstr: The docstring for the current Scope. :param docstr: The docstring for the current Scope.
:type docstr: str :type docstr: str
""" """
def __init__(self, name, supers, indent, line_nr, docstr=''): def __init__(self, name, supers, start_pos, docstr=''):
super(Class, self).__init__(indent, line_nr, docstr) super(Class, self).__init__(start_pos, docstr)
self.name = name self.name = name
name.parent = self name.parent = self
self.supers = supers self.supers = supers
@@ -300,16 +294,14 @@ class Function(Scope):
:param name: The Function name. :param name: The Function name.
:type name: string :type name: string
:param params: The parameters (Statement) of a Function. :param params: The parameters (Statement) of a Function.
:type name: list :type params: list
:param indent: The indent level of the flow statement. :param start_pos: The start position (line, column) the Function.
:type indent: int :type start_pos: tuple(int, int)
:param line_nr: Line number of the flow statement.
:type line_nr: int
:param docstr: The docstring for the current Scope. :param docstr: The docstring for the current Scope.
:type docstr: str :type docstr: str
""" """
def __init__(self, name, params, indent, line_nr, docstr=''): def __init__(self, name, params, start_pos, docstr=''):
Scope.__init__(self, indent, line_nr, docstr) Scope.__init__(self, start_pos, docstr)
self.name = name self.name = name
name.parent = self name.parent = self
self.params = params self.params = params
@@ -356,17 +348,15 @@ class Flow(Scope):
:type command: str :type command: str
:param inits: The initializations of a flow -> while 'statement'. :param inits: The initializations of a flow -> while 'statement'.
:type inits: list(Statement) :type inits: list(Statement)
:param indent: The indent level of the flow statement. :param start_pos: Position (line, column) of the Flow statement.
:type indent: int :type start_pos: tuple(int, int)
:param line_nr: Line number of the flow statement.
:type line_nr: int
:param set_vars: Local variables used in the for loop (only there). :param set_vars: Local variables used in the for loop (only there).
:type set_vars: list :type set_vars: list
""" """
def __init__(self, command, inits, indent, line_nr, set_vars=None): def __init__(self, command, inits, start_pos, set_vars=None):
self._parent = None self._parent = None
self.next = None self.next = None
super(Flow, self).__init__(indent, line_nr, '') super(Flow, self).__init__(start_pos, '')
self.command = command self.command = command
# These have to be statements, because of with, which takes multiple. # These have to be statements, because of with, which takes multiple.
self.inits = inits self.inits = inits
@@ -445,8 +435,8 @@ class ForFlow(Flow):
""" """
Used for the for loop, because there are two statement parts. Used for the for loop, because there are two statement parts.
""" """
def __init__(self, command, inits, indent, line_nr, set_stmt): def __init__(self, command, inits, start_pos, set_stmt):
super(ForFlow, self).__init__(command, inits, indent, line_nr, super(ForFlow, self).__init__(command, inits, start_pos,
set_stmt.used_vars) set_stmt.used_vars)
self.set_stmt = set_stmt self.set_stmt = set_stmt
@@ -458,8 +448,8 @@ class Import(Simple):
>>> 1+1 >>> 1+1
2 2
:param line_nr: Line number. :param start_pos: Position (line, column) of the Import.
:type line_nr: int :type start_pos: tuple(int, int)
:param namespace: The import, can be empty if a star is given :param namespace: The import, can be empty if a star is given
:type namespace: Name :type namespace: Name
:param alias: The alias of a namespace(valid in the current namespace). :param alias: The alias of a namespace(valid in the current namespace).
@@ -469,9 +459,9 @@ class Import(Simple):
:param star: If a star is used -> from time import *. :param star: If a star is used -> from time import *.
:type star: bool :type star: bool
""" """
def __init__(self, indent, line_nr, line_end, namespace, alias='', \ def __init__(self, start_pos, end_pos, namespace, alias='', \
from_ns='', star=False, relative_count=None): from_ns='', star=False, relative_count=None):
super(Import, self).__init__(indent, line_nr, line_end) super(Import, self).__init__(start_pos, end_pos)
self.namespace = namespace self.namespace = namespace
if namespace: if namespace:
@@ -523,14 +513,12 @@ class Statement(Simple):
:param used_vars: str :param used_vars: str
:param token_list: Token list which is also peppered with Name. :param token_list: Token list which is also peppered with Name.
:param token_list: list :param token_list: list
:param indent: The indent level of the flow statement. :param start_pos: Position (line, column) of the Statement.
:type indent: int :type start_pos: tuple(int, int)
:param line_nr: Line number of the flow statement.
:type line_nr: int
""" """
def __init__(self, code, set_vars, used_funcs, used_vars, token_list, def __init__(self, code, set_vars, used_funcs, used_vars, token_list,
indent, line_nr, line_end): start_pos, end_pos):
super(Statement, self).__init__(indent, line_nr, line_end) super(Statement, self).__init__(start_pos, end_pos)
self.code = code self.code = code
self.set_vars = set_vars self.set_vars = set_vars
self.used_funcs = used_funcs self.used_funcs = used_funcs
@@ -591,7 +579,7 @@ class Statement(Simple):
for i, tok_temp in tok_iter: for i, tok_temp in tok_iter:
#print 'tok', tok_temp, result #print 'tok', tok_temp, result
try: try:
token_type, tok, indent = tok_temp token_type, tok = tok_temp
except TypeError: except TypeError:
# the token is a Name, which has already been parsed # the token is a Name, which has already been parsed
tok = tok_temp tok = tok_temp
@@ -712,11 +700,12 @@ class Param(Statement):
But this is not to define function calls. But this is not to define function calls.
""" """
def __init__(self, code, set_vars, used_funcs, used_vars, token_list, def __init__(self, code, set_vars, used_funcs, used_vars, token_list,
indent, line_nr, line_end): start_pos, end_pos):
super(Param, self).__init__(code, set_vars, used_funcs, super(Param, self).__init__(code, set_vars, used_funcs,
used_vars, token_list, indent, line_nr, line_end) used_vars, token_list, start_pos, end_pos)
# this is defined by the parser later on, not at the initialization # this is defined by the parser later on, not at the initialization
# it is the position in the call (first argument, second...)
self.position = None self.position = None
def get_name(self): def get_name(self):
@@ -904,8 +893,8 @@ class Name(Simple):
So a name like "module.class.function" So a name like "module.class.function"
would result in an array of [module, class, function] would result in an array of [module, class, function]
""" """
def __init__(self, names, indent, line_nr, line_end): def __init__(self, names, start_pos, end_pos):
super(Name, self).__init__(indent, line_nr, line_end) super(Name, self).__init__(start_pos, end_pos)
self.names = tuple(NamePart(n) for n in names) self.names = tuple(NamePart(n) for n in names)
def get_code(self): def get_code(self):
@@ -916,15 +905,14 @@ class Name(Simple):
return self.get_code() return self.get_code()
def __eq__(self, other): def __eq__(self, other):
return self.names == other.names \ return self.names == other.names and self.start_pos == other.end_pos
and self.indent == other.indent \
and self.line_nr == self.line_nr
def __ne__(self, other): def __ne__(self, other):
return not self.__eq__(other) return not self.__eq__(other)
def __hash__(self): def __hash__(self):
return hash(self.names) + hash(self.indent) + hash(self.line_nr) return hash(self.names) + hash(self.start_pos[0]) \
+ hash(self.start_pos[1])
def __len__(self): def __len__(self):
return len(self.names) return len(self.names)
@@ -938,7 +926,7 @@ class PyFuzzyParser(object):
:param code: The codebase for the parser. :param code: The codebase for the parser.
:type code: str :type code: str
:param user_position: The line/column, the user is currently on. :param user_position: The line/column, the user is currently on.
:type user_position: tuple(line, column) :type user_position: tuple(int, int)
""" """
def __init__(self, code, module_path=None, user_position=(None,None)): def __init__(self, code, module_path=None, user_position=(None,None)):
self.user_position = user_position self.user_position = user_position
@@ -949,7 +937,9 @@ class PyFuzzyParser(object):
self.scope = self.top self.scope = self.top
self.current = (None, None, None) self.current = (None, None, None)
self._tokenize_line_nr = 0 # Stuff to fix tokenize errors. The parser is pretty good in tolerating
# any errors of tokenize and just parse ahead.
self._tokenize_start_pos = 0
self._line_of_tokenize_restart = 0 self._line_of_tokenize_restart = 0
self.parse() self.parse()
@@ -961,38 +951,44 @@ class PyFuzzyParser(object):
return "<%s: %s>" % (self.__class__.__name__, self.top) return "<%s: %s>" % (self.__class__.__name__, self.top)
@property @property
def line_nr(self): def start_pos(self):
return self._line_of_tokenize_restart + self._tokenize_line_nr return (self._line_of_tokenize_restart + self._tokenize_start_pos[0],
self._tokenize_start_pos[1])
@property
def end_pos(self):
return (self._line_of_tokenize_restart + self._tokenize_end_pos[0],
self._tokenize_end_pos[1])
def _parsedotname(self, pre_used_token=None): def _parsedotname(self, pre_used_token=None):
""" """
The dot name parser parses a name, variable or function and returns The dot name parser parses a name, variable or function and returns
their names. their names.
:return: list of the names, token_type, nexttoken, start_indent, \ :return: Tuple of Name, token_type, nexttoken.
start_line. :rtype: tuple(Name, int, str)
:rtype: (Name, int, str, int, int)
""" """
names = [] names = []
if pre_used_token is None: if pre_used_token is None:
token_type, tok, indent = self.next() token_type, tok = self.next()
start_line = self.line_nr
if token_type != tokenize.NAME and tok != '*': if token_type != tokenize.NAME and tok != '*':
return ([], token_type, tok, indent, start_line) return ([], token_type, tok, self.start_pos)
else: else:
token_type, tok, indent = pre_used_token token_type, tok = pre_used_token
start_line = self.line_nr
names.append(tok) names.append(tok)
start_indent = indent first_pos = self.start_pos
while True: while True:
token_type, tok, indent = self.next() token_type, tok = self.next()
if tok != '.': if tok != '.':
break break
token_type, tok, indent = self.next() token_type, tok = self.next()
if token_type != tokenize.NAME: if token_type != tokenize.NAME:
break break
names.append(tok) names.append(tok)
return (names, token_type, tok, start_indent, start_line)
n = Name(names, first_pos, self.end_pos) if names else None
return (n, token_type, tok)
def _parseimportlist(self): def _parseimportlist(self):
""" """
@@ -1015,25 +1011,21 @@ class PyFuzzyParser(object):
continue_kw = [",", ";", "\n", ')'] \ continue_kw = [",", ";", "\n", ')'] \
+ list(set(keyword.kwlist) - set(['as'])) + list(set(keyword.kwlist) - set(['as']))
while True: while True:
token_type, tok, indent = self.next() token_type, tok = self.next()
if brackets and tok == '\n': if brackets and tok == '\n':
self.next() self.next()
if tok == '(': # python allows only one `(` in the statement. if tok == '(': # python allows only one `(` in the statement.
brackets = True brackets = True
self.next() self.next()
name, token_type, tok, start_indent, start_line = \ i, token_type, tok = self._parsedotname(self.current)
self._parsedotname(self.current) if not i:
if not name:
break break
name2 = None name2 = None
if tok == 'as': if tok == 'as':
name2, token_type, tok, start_indent2, start_line = \ name2, token_type, tok = self._parsedotname()
self._parsedotname()
name2 = Name(name2, start_indent2, start_line, self.line_nr)
i = Name(name, start_indent, start_line, self.line_nr)
imports.append((i, name2)) imports.append((i, name2))
while tok not in continue_kw: while tok not in continue_kw:
token_type, tok, indent = self.next() token_type, tok = self.next()
if not (tok == "," or brackets and tok == '\n'): if not (tok == "," or brackets and tok == '\n'):
break break
return imports return imports
@@ -1059,7 +1051,7 @@ class PyFuzzyParser(object):
return names return names
def _parsefunction(self, indent): def _parsefunction(self):
""" """
The parser for a text functions. Process the tokens, which follow a The parser for a text functions. Process the tokens, which follow a
function definition. function definition.
@@ -1067,25 +1059,25 @@ class PyFuzzyParser(object):
:return: Return a Scope representation of the tokens. :return: Return a Scope representation of the tokens.
:rtype: Function :rtype: Function
""" """
start_line = self.line_nr first_pos = self.start_pos
token_type, fname, ind = self.next() token_type, fname = self.next()
if token_type != tokenize.NAME: if token_type != tokenize.NAME:
return None return None
fname = Name([fname], ind, self.line_nr, self.line_nr) fname = Name([fname], self.start_pos, self.end_pos)
token_type, open, ind = self.next() token_type, open = self.next()
if open != '(': if open != '(':
return None return None
params = self._parseparen() params = self._parseparen()
token_type, colon, ind = self.next() token_type, colon = self.next()
if colon != ':': if colon != ':':
return None return None
return Function(fname, params, indent, start_line) return Function(fname, params, first_pos)
def _parseclass(self, indent): def _parseclass(self):
""" """
The parser for a text class. Process the tokens, which follow a The parser for a text class. Process the tokens, which follow a
class definition. class definition.
@@ -1093,26 +1085,26 @@ class PyFuzzyParser(object):
:return: Return a Scope representation of the tokens. :return: Return a Scope representation of the tokens.
:rtype: Class :rtype: Class
""" """
start_line = self.line_nr first_pos = self.start_pos
token_type, cname, ind = self.next() token_type, cname = self.next()
if token_type != tokenize.NAME: if token_type != tokenize.NAME:
debug.dbg("class: syntax error - token is not a name@%s (%s: %s)" \ debug.dbg("class: syntax error - token is not a name@%s (%s: %s)"
% (self.line_nr, tokenize.tok_name[token_type], cname)) % (self.start_pos[0], tokenize.tok_name[token_type], cname))
return None return None
cname = Name([cname], ind, self.line_nr, self.line_nr) cname = Name([cname], self.start_pos, self.end_pos)
super = [] super = []
token_type, next, ind = self.next() token_type, next = self.next()
if next == '(': if next == '(':
super = self._parseparen() super = self._parseparen()
token_type, next, ind = self.next() token_type, next = self.next()
if next != ':': if next != ':':
debug.dbg("class: syntax error - %s@%s" % (cname, self.line_nr)) debug.dbg("class syntax: %s@%s" % (cname, self.start_pos[0]))
return None return None
return Class(cname, super, indent, start_line) return Class(cname, super, first_pos)
def _parse_statement(self, pre_used_token=None, added_breaks=None, def _parse_statement(self, pre_used_token=None, added_breaks=None,
stmt_class=Statement): stmt_class=Statement):
@@ -1122,7 +1114,7 @@ class PyFuzzyParser(object):
>>> a = test(b) >>> a = test(b)
>>> a += 3 - 2 or b >>> a += 3 - 2 or b
and so on. One row at a time. and so on. One line at a time.
:param pre_used_token: The pre parsed token. :param pre_used_token: The pre parsed token.
:type pre_used_token: set :type pre_used_token: set
@@ -1140,11 +1132,11 @@ class PyFuzzyParser(object):
is_return = None is_return = None
if pre_used_token: if pre_used_token:
token_type, tok, indent = pre_used_token token_type, tok = pre_used_token
else: else:
token_type, tok, indent = self.next() token_type, tok = self.next()
line_start = self.line_nr first_pos = self.start_pos
# the difference between "break" and "always break" is that the latter # the difference between "break" and "always break" is that the latter
# will even break in parentheses. This is true for typical flow # will even break in parentheses. This is true for typical flow
@@ -1163,14 +1155,13 @@ class PyFuzzyParser(object):
tok_list.append(self.current) tok_list.append(self.current)
if tok == 'as': if tok == 'as':
string += " %s " % tok string += " %s " % tok
token_type, tok, indent_dummy = self.next() token_type, tok = self.next()
if token_type == tokenize.NAME: if token_type == tokenize.NAME:
path, token_type, tok, start_indent, start_line = \ n, token_type, tok = self._parsedotname(self.current)
self._parsedotname(self.current) if n:
n = Name(path, start_indent, start_line, self.line_nr) set_vars.append(n)
set_vars.append(n)
tok_list.append(n) tok_list.append(n)
string += ".".join(path) string += ".".join(n.names)
continue continue
elif token_type == tokenize.NAME: elif token_type == tokenize.NAME:
#print 'is_name', tok #print 'is_name', tok
@@ -1181,8 +1172,8 @@ class PyFuzzyParser(object):
# statement. This is a hack, but I could not come up # statement. This is a hack, but I could not come up
# with a better solution. # with a better solution.
# This is basically a reset of the statement. # This is basically a reset of the statement.
debug.warning('keyword in statement @%s', tok_list, debug.warning('keyword in statement %s@%s', tok_list,
self.line_nr) self.start_pos[0])
tok_list = [self.current] tok_list = [self.current]
set_vars = [] set_vars = []
used_funcs = [] used_funcs = []
@@ -1192,12 +1183,12 @@ class PyFuzzyParser(object):
if tok in ['return', 'yield']: if tok in ['return', 'yield']:
is_return = tok is_return = tok
elif tok in ['print', 'exec']: elif tok in ['print', 'exec']:
# TODO they should be reinstated, since the goal of the
# parser is a different one.
# delete those statements, just let the rest stand there # delete those statements, just let the rest stand there
set_string = '' set_string = ''
else: else:
path, token_type, tok, start_indent, start_line = \ n, token_type, tok = self._parsedotname(self.current)
self._parsedotname(self.current)
n = Name(path, start_indent, start_line, self.line_nr)
tok_list.pop() # removed last entry, because we add Name tok_list.pop() # removed last entry, because we add Name
tok_list.append(n) tok_list.append(n)
if tok == '(': if tok == '(':
@@ -1207,7 +1198,7 @@ class PyFuzzyParser(object):
used_vars.append(n) used_vars.append(n)
if string and re.match(r'[\w\d\'"]', string[-1]): if string and re.match(r'[\w\d\'"]', string[-1]):
string += ' ' string += ' '
string += ".".join(path) string += ".".join(n.names)
#print 'parse_stmt', tok, tokenize.tok_name[token_type] #print 'parse_stmt', tok, tokenize.tok_name[token_type]
continue continue
elif '=' in tok and not tok in ['>=', '<=', '==', '!=']: elif '=' in tok and not tok in ['>=', '<=', '==', '!=']:
@@ -1224,9 +1215,7 @@ class PyFuzzyParser(object):
string = set_string string = set_string
else: else:
string += tok string += tok
# caution: don't use indent anywhere, token_type, tok = self.next()
# it's not working with the name parsing
token_type, tok, indent_dummy = self.next()
if not string: if not string:
return None, tok return None, tok
#print 'new_stat', string, set_vars, used_funcs, used_vars #print 'new_stat', string, set_vars, used_funcs, used_vars
@@ -1234,8 +1223,8 @@ class PyFuzzyParser(object):
and self.last_token[1] == tokenize.STRING: and self.last_token[1] == tokenize.STRING:
self.scope.add_docstr(self.last_token[1]) self.scope.add_docstr(self.last_token[1])
else: else:
stmt = stmt_class(string, set_vars, used_funcs, used_vars,\ stmt = stmt_class(string, set_vars, used_funcs, used_vars, \
tok_list, indent, line_start, self.line_nr) tok_list, first_pos, self.end_pos)
if is_return: if is_return:
# add returns to the scope # add returns to the scope
func = self.scope.get_parent_until(Function) func = self.scope.get_parent_until(Function)
@@ -1250,14 +1239,14 @@ class PyFuzzyParser(object):
def next(self): def next(self):
""" Generate the next tokenize pattern. """ """ Generate the next tokenize pattern. """
type, tok, position, dummy, self.parserline = next(self.gen) type, tok, self._tokenize_start_pos, self._tokenize_end_pos, \
(self._tokenize_line_nr, indent) = position self.parserline = next(self.gen)
if self.line_nr == self.user_position[0]: if self.start_pos[0] == self.user_position[0]:
debug.dbg('user scope found [%s] =%s' % \ debug.dbg('user scope found [%s] =%s' % \
(self.parserline.replace('\n', ''), repr(self.scope))) (self.parserline.replace('\n', ''), repr(self.scope)))
self.user_scope = self.scope self.user_scope = self.scope
self.last_token = self.current self.last_token = self.current
self.current = (type, tok, indent) self.current = (type, tok)
return self.current return self.current
def parse(self): def parse(self):
@@ -1282,44 +1271,43 @@ class PyFuzzyParser(object):
self.freshscope = True self.freshscope = True
while True: while True:
try: try:
token_type, tok, indent = self.next() token_type, tok = self.next()
#debug.dbg('main: tok=[%s] type=[%s] indent=[%s]'\ #debug.dbg('main: tok=[%s] type=[%s] indent=[%s]'\
# % (tok, token_type, indent)) # % (tok, token_type, start_position[0]))
while token_type == tokenize.DEDENT and self.scope != self.top: while token_type == tokenize.DEDENT and self.scope != self.top:
debug.dbg('dedent', self.scope) debug.dbg('dedent', self.scope)
token_type, tok, indent = self.next() token_type, tok = self.next()
if indent <= self.scope.indent: if self.start_pos[1] <= self.scope.start_pos[1]:
self.scope.line_end = self.line_nr self.scope.end_pos = self.start_pos
self.scope = self.scope.parent self.scope = self.scope.parent
# check again for unindented stuff. this is true for syntax # check again for unindented stuff. this is true for syntax
# errors. only check for names, because thats relevant here. If # errors. only check for names, because thats relevant here. If
# some docstrings are not indented, I don't care. # some docstrings are not indented, I don't care.
while indent <= self.scope.indent \ while self.start_pos[1] <= self.scope.start_pos[1] \
and (token_type == tokenize.NAME or tok in ['(', '['])\ and (token_type == tokenize.NAME or tok in ['(', '['])\
and self.scope != self.top: and self.scope != self.top:
debug.dbg('syntax: dedent @%s - %s<=%s', \ debug.dbg('syntax: dedent @%s - %s<=%s', self.start_pos)
(self.line_nr, indent, self.scope.indent)) self.scope.end_pos = self.start_pos
self.scope.line_end = self.line_nr
self.scope = self.scope.parent self.scope = self.scope.parent
start_line = self.line_nr first_pos = self.start_pos
if tok == 'def': if tok == 'def':
func = self._parsefunction(indent) func = self._parsefunction()
if func is None: if func is None:
debug.warning("function: syntax error@%s" % debug.warning("function: syntax error@%s" %
self.line_nr) self.start_pos[0])
continue continue
debug.dbg("new scope: function %s" % (func.name)) debug.dbg("new scope: function %s" % (func.name))
self.freshscope = True self.freshscope = True
self.scope = self.scope.add_scope(func, decorators) self.scope = self.scope.add_scope(func, decorators)
decorators = [] decorators = []
elif tok == 'class': elif tok == 'class':
cls = self._parseclass(indent) cls = self._parseclass()
if cls is None: if cls is None:
debug.warning("class: syntax error@%s" % debug.warning("class: syntax error@%s" %
self.line_nr) self.start_pos[0])
continue continue
self.freshscope = True self.freshscope = True
debug.dbg("new scope: class %s" % (cls.name)) debug.dbg("new scope: class %s" % (cls.name))
@@ -1329,7 +1317,7 @@ class PyFuzzyParser(object):
elif tok == 'import': elif tok == 'import':
imports = self._parseimportlist() imports = self._parseimportlist()
for m, alias in imports: for m, alias in imports:
i = Import(indent, start_line, self.line_nr, m, alias) i = Import(first_pos, self.end_pos, m, alias)
self.scope.add_import(i) self.scope.add_import(i)
debug.dbg("new import: %s" % (i), self.current) debug.dbg("new import: %s" % (i), self.current)
self.freshscope = False self.freshscope = False
@@ -1337,24 +1325,22 @@ class PyFuzzyParser(object):
# take care for relative imports # take care for relative imports
relative_count = 0 relative_count = 0
while 1: while 1:
token_type, tok, indent = self.next() token_type, tok = self.next()
if tok != '.': if tok != '.':
break break
relative_count += 1 relative_count += 1
# the from import # the from import
mod, token_type, tok, start_indent, start_line2 = \ mod, token_type, tok = self._parsedotname(self.current)
self._parsedotname(self.current)
if not mod or tok != "import": if not mod or tok != "import":
debug.warning("from: syntax error@%s" % debug.warning("from: syntax error@%s" %
self.line_nr) self.start_pos[0])
continue continue
mod = Name(mod, start_indent, start_line2, self.line_nr)
names = self._parseimportlist() names = self._parseimportlist()
for name, alias in names: for name, alias in names:
star = name.names[0] == '*' star = name.names[0] == '*'
if star: if star:
name = None name = None
i = Import(indent, start_line, self.line_nr, name, i = Import(first_pos, self.end_pos, name,
alias, mod, star, relative_count) alias, mod, star, relative_count)
self.scope.add_import(i) self.scope.add_import(i)
debug.dbg("new from: %s" % (i)) debug.dbg("new from: %s" % (i))
@@ -1365,9 +1351,9 @@ class PyFuzzyParser(object):
if tok == 'in': if tok == 'in':
statement, tok = self._parse_statement() statement, tok = self._parse_statement()
if tok == ':': if tok == ':':
f = ForFlow('for', [statement], indent, f = ForFlow('for', [statement], first_pos,
self.line_nr, set_stmt) set_stmt)
debug.dbg("new scope: flow for@%s" % (f.line_nr)) debug.dbg("new scope: forflow@%s" % f.start_pos[0])
self.scope = self.scope.add_statement(f) self.scope = self.scope.add_statement(f)
elif tok in ['if', 'while', 'try', 'with'] + extended_flow: elif tok in ['if', 'while', 'try', 'with'] + extended_flow:
@@ -1384,20 +1370,18 @@ class PyFuzzyParser(object):
if command == 'except' and tok in added_breaks: if command == 'except' and tok in added_breaks:
# the except statement defines a var # the except statement defines a var
# this is only true for python 2 # this is only true for python 2
path, token_type, tok, start_indent, start_line2 =\ n, token_type, tok = self._parsedotname()
self._parsedotname() if n:
n = Name(path, start_indent, start_line2, statement.set_vars.append(n)
self.line_nr) statement.code += ',' + n.get_code()
statement.set_vars.append(n)
statement.code += ',' + n.get_code()
if statement: if statement:
inits.append(statement) inits.append(statement)
first = False first = False
if tok == ':': if tok == ':':
f = Flow(command, inits, indent, self.line_nr) f = Flow(command, inits, first_pos)
debug.dbg("new scope: flow %s@%s" debug.dbg("new scope: flow %s@%s"
% (command, self.line_nr)) % (command, self.start_pos[0]))
if command in extended_flow: if command in extended_flow:
# the last statement has to be another part of # the last statement has to be another part of
# the flow statement, because a dedent releases the # the flow statement, because a dedent releases the
@@ -1412,7 +1396,7 @@ class PyFuzzyParser(object):
self.scope = s self.scope = s
else: else:
debug.warning('syntax err, flow started @%s', debug.warning('syntax err, flow started @%s',
self.line_nr) self.start_pos[0])
# globals # globals
elif tok == 'global': elif tok == 'global':
stmt, tok = self._parse_statement(self.current) stmt, tok = self._parse_statement(self.current)
@@ -1446,7 +1430,7 @@ class PyFuzzyParser(object):
tokenize.NEWLINE, tokenize.NL, tokenize.NEWLINE, tokenize.NL,
tokenize.ENDMARKER]: tokenize.ENDMARKER]:
debug.warning('token not classified', tok, token_type, debug.warning('token not classified', tok, token_type,
self.line_nr) self.start_pos[0])
except StopIteration: # thrown on EOF except StopIteration: # thrown on EOF
break break
except tokenize.TokenError: except tokenize.TokenError:
@@ -1459,9 +1443,10 @@ class PyFuzzyParser(object):
# and restarts the parser. # and restarts the parser.
# (This is a rather unlikely error message, for normal code, # (This is a rather unlikely error message, for normal code,
# tokenize seems to be pretty tolerant) # tokenize seems to be pretty tolerant)
self._line_of_tokenize_restart = self.line_nr + 1 self._line_of_tokenize_restart = self.start_pos[0] + 1
self._tokenize_line_nr = 0 self._tokenize_start_pos = (0,0)
self._tokenize_end_pos = (0,0)
debug.warning('indentation error on line %s, ignoring it' % debug.warning('indentation error on line %s, ignoring it' %
(self.line_nr)) (self.start_pos[0]))
self.gen = tokenize.generate_tokens(buf.readline) self.gen = tokenize.generate_tokens(buf.readline)
return self.top return self.top