VimPlug/jedi-fork
1
0
Fork 0
forked from VimPlug/jedi
Code Pull Requests Activity

* made Token readonly like a tuple by using @property

Browse Source 
* end_pos calculation didn't respect multiline tokens
* replaced all index access to Token
* wrapped all code that injects token tuples with
  Token.from_tuple()
* repr of Token is still its tuple form!?
* PEP8 where I read or wrote code
This commit is contained in:
Jean-Louis Fuchs
2013-12-08 01:32:58 +01:00
parent 13680945d6
commit 3204a39f6c
5 changed files with 122 additions and 70 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
jedi/interpret.py
View File

@@ -6,6 +6,7 @@ import itertools
import tokenize import tokenize
from jedi.parser import representation as pr from jedi.parser import representation as pr
from jedi.parser import token
class ObjectImporter(object): class ObjectImporter(object):
@@ -156,11 +157,13 @@ class ObjectImporter(object):
names=[(rhs, (0, 0))], names=[(rhs, (0, 0))],
start_pos=(0, 0), start_pos=(0, 0),
end_pos=(None, None)) end_pos=(None, None))
token_list = [lhsname, (tokenize.OP, '=', (0, 0)), rhsname] token_list = [lhsname, token.Token.from_tuple(
(tokenize.OP, '=', (0, 0))
), rhsname]
if call: if call:
token_list.extend([ token_list.extend([
(tokenize.OP, '(', (0, 0)), token.Token.from_tuple((tokenize.OP, '(', (0, 0))),
(tokenize.OP, ')', (0, 0)), token.Token.from_tuple((tokenize.OP, ')', (0, 0))),
]) ])
return pr.Statement( return pr.Statement(
module=submodule, module=submodule,

8
jedi/parser/__init__.py
View File

@@ -392,18 +392,18 @@ class Parser(object):
first_tok = tok_list[0] first_tok = tok_list[0]
# docstrings # docstrings
if len(tok_list) == 1 and not isinstance(first_tok, pr.Name) \ if len(tok_list) == 1 and not isinstance(first_tok, pr.Name) \
and first_tok[0] == tokenize.STRING: and first_tok.token_type == tokenize.STRING:
# Normal docstring check # Normal docstring check
if self.freshscope and not self.no_docstr: if self.freshscope and not self.no_docstr:
self._scope.add_docstr(first_tok[1]) self._scope.add_docstr(first_tok.token)
return None, tok return None, tok
# Attribute docstring (PEP 224) support (sphinx uses it, e.g.) # Attribute docstring (PEP 224) support (sphinx uses it, e.g.)
# If string literal is being parsed... # If string literal is being parsed...
elif first_tok[0] == tokenize.STRING: elif first_tok.token_type == tokenize.STRING:
with common.ignored(IndexError, AttributeError): with common.ignored(IndexError, AttributeError):
# ...then set it as a docstring # ...then set it as a docstring
self._scope.statements[-1].add_docstr(first_tok[1]) self._scope.statements[-1].add_docstr(first_tok.token)
return None, tok return None, tok
stmt = stmt_class(self.module, tok_list, first_pos, self.end_pos, stmt = stmt_class(self.module, tok_list, first_pos, self.end_pos,

66
jedi/parser/representation.py
View File

@@ -898,7 +898,7 @@ class Statement(Simple):
c = token_iterator.current[1] c = token_iterator.current[1]
arr.end_pos = c.end_pos if isinstance(c, Simple) \ arr.end_pos = c.end_pos if isinstance(c, Simple) \
else (c[2][0], c[2][1] + len(c[1])) else c.end_pos
return arr, break_tok return arr, break_tok
def parse_stmt(token_iterator, maybe_dict=False, added_breaks=(), def parse_stmt(token_iterator, maybe_dict=False, added_breaks=(),
@@ -920,9 +920,10 @@ class Statement(Simple):
# it's not possible to set it earlier # it's not possible to set it earlier
tok.parent = self tok.parent = self
else: else:
token_type, tok, start_tok_pos = tok_temp tok = tok_temp.token
last_end_pos = end_pos start_tok_pos = tok_temp.start_pos
end_pos = start_tok_pos[0], start_tok_pos[1] + len(tok) last_end_pos = end_pos
end_pos = tok_temp.end_pos
if first: if first:
first = False first = False
start_pos = start_tok_pos start_pos = start_tok_pos
@@ -932,8 +933,12 @@ class Statement(Simple):
if lambd is not None: if lambd is not None:
token_list.append(lambd) token_list.append(lambd)
elif tok == 'for': elif tok == 'for':
list_comp, tok = parse_list_comp(token_iterator, list_comp, tok = parse_list_comp(
token_list, start_pos, last_end_pos) token_iterator,
token_list,
start_pos,
last_end_pos
)
if list_comp is not None: if list_comp is not None:
token_list = [list_comp] token_list = [list_comp]
@@ -944,9 +949,12 @@ class Statement(Simple):
if level == 0 and tok in closing_brackets \ if level == 0 and tok in closing_brackets \
or tok in added_breaks \ or tok in added_breaks \
or level == 1 and (tok == ',' or level == 1 and (
or maybe_dict and tok == ':' tok == ','
or is_assignment(tok) and break_on_assignment): or maybe_dict and tok == ':'
or is_assignment(tok)
and break_on_assignment
):
end_pos = end_pos[0], end_pos[1] - 1 end_pos = end_pos[0], end_pos[1] - 1
break break
token_list.append(tok_temp) token_list.append(tok_temp)
@@ -954,8 +962,14 @@ class Statement(Simple):
if not token_list: if not token_list:
return None, tok return None, tok
statement = stmt_class(self._sub_module, token_list, statement = stmt_class(
start_pos, end_pos, self.parent, set_name_parents=False) self._sub_module,
token_list,
start_pos,
end_pos,
self.parent,
set_name_parents=False
)
return statement, tok return statement, tok
def parse_lambda(token_iterator): def parse_lambda(token_iterator):
@@ -984,8 +998,9 @@ class Statement(Simple):
return lambd, tok return lambd, tok
def parse_list_comp(token_iterator, token_list, start_pos, end_pos): def parse_list_comp(token_iterator, token_list, start_pos, end_pos):
def parse_stmt_or_arr(token_iterator, added_breaks=(), def parse_stmt_or_arr(
names_are_set_vars=False): token_iterator, added_breaks=(), names_are_set_vars=False
):
stmt, tok = parse_stmt(token_iterator, stmt, tok = parse_stmt(token_iterator,
added_breaks=added_breaks) added_breaks=added_breaks)
if not stmt: if not stmt:
@@ -1039,12 +1054,16 @@ class Statement(Simple):
start_pos = tok.start_pos start_pos = tok.start_pos
end_pos = tok.end_pos end_pos = tok.end_pos
else: else:
token_type, tok, start_pos = tok_temp token_type = tok_temp.token_type
end_pos = start_pos[0], start_pos[1] + len(tok) tok = tok_temp.token
start_pos = tok_temp.start_pos
end_pos = tok_temp.end_pos
if is_assignment(tok): if is_assignment(tok):
# This means, there is an assignment here. # This means, there is an assignment here.
# Add assignments, which can be more than one # Add assignments, which can be more than one
self._assignment_details.append((result, tok)) self._assignment_details.append(
(result, tok_temp.token)
)
result = [] result = []
is_chain = False is_chain = False
continue continue
@@ -1072,8 +1091,9 @@ class Statement(Simple):
result.append(call) result.append(call)
is_chain = False is_chain = False
elif tok in brackets.keys(): elif tok in brackets.keys():
arr, is_ass = parse_array(token_iterator, brackets[tok], arr, is_ass = parse_array(
start_pos) token_iterator, brackets[tok], start_pos
)
if result and isinstance(result[-1], StatementElement): if result and isinstance(result[-1], StatementElement):
result[-1].set_execution(arr) result[-1].set_execution(arr)
else: else:
@@ -1098,8 +1118,14 @@ class Statement(Simple):
e = (t[2][0], t[2][1] + len(t[1])) \ e = (t[2][0], t[2][1] + len(t[1])) \
if isinstance(t, tuple) else t.start_pos if isinstance(t, tuple) else t.start_pos
stmt = Statement(self._sub_module, result, stmt = Statement(
start_pos, e, self.parent, set_name_parents=False) self._sub_module,
result,
start_pos,
e,
self.parent,
set_name_parents=False
)
stmt._commands = result stmt._commands = result
arr, break_tok = parse_array(token_iterator, Array.TUPLE, arr, break_tok = parse_array(token_iterator, Array.TUPLE,
stmt.start_pos, stmt) stmt.start_pos, stmt)

84
jedi/parser/token.py
View File

@@ -27,18 +27,10 @@ class Token(object):
>>> a.__setstate__((1, 2, 3, 4)) >>> a.__setstate__((1, 2, 3, 4))
>>> a >>> a
<Token: (1, 2, (3, 4))> <Token: (1, 2, (3, 4))>
>>> a[2] = (2, 1)
>>> a
<Token: (1, 2, (2, 1))>
>>> a.start_pos
(2, 1)
>>> a.token
2
>>> a.start_pos = (3, 4)
>>> a
<Token: (1, 2, (3, 4))>
>>> a.start_pos >>> a.start_pos
(3, 4) (3, 4)
>>> a.token
2
>>> a.start_pos_col >>> a.start_pos_col
4 4
>>> Token.from_tuple((6, 5, (4, 3))) >>> Token.from_tuple((6, 5, (4, 3)))
@@ -47,7 +39,7 @@ class Token(object):
True True
""" """
__slots__ = [ __slots__ = [
"token_type", "token", "start_pos_line", "start_pos_col" "_token_type", "_token", "_start_pos_line", "_start_pos_col"
] ]
@classmethod @classmethod
@@ -57,10 +49,10 @@ class Token(object):
def __init__( def __init__(
self, token_type, token, start_pos_line, start_pos_col self, token_type, token, start_pos_line, start_pos_col
): ):
self.token_type = token_type self._token_type = token_type
self.token = token self._token = token
self.start_pos_line = start_pos_line self._start_pos_line = start_pos_line
self.start_pos_col = start_pos_col self._start_pos_col = start_pos_col
def __repr__(self): def __repr__(self):
return "<%s: %s>" % (type(self).__name__, tuple(self)) return "<%s: %s>" % (type(self).__name__, tuple(self))
@@ -85,34 +77,40 @@ class Token(object):
else: else:
raise IndexError("list index out of range") raise IndexError("list index out of range")
# Backward compatibility @property
def __setitem__(self, key, value): def token_type(self):
# setitem analogous to getitem return self._token_type
if key == 0:
self.token_type = value @property
elif key == 1: def token(self):
self.token = value return self._token
elif key == 2:
self.start_pos_line = value[0] @property
self.start_pos_col = value[1] def start_pos_line(self):
else: return self._start_pos_line
raise IndexError("list index out of range")
@property
def start_pos_col(self):
return self._start_pos_col
# Backward compatibility # Backward compatibility
def __getattr__(self, attr): @property
# Expose the missing start_pos attribute def start_pos(self):
if attr == "start_pos": return (self.start_pos_line, self.start_pos_col)
return (self.start_pos_line, self.start_pos_col)
else:
return object.__getattr__(self, attr)
def __setattr__(self, attr, value): @property
# setattr analogous to getattr for symmetry def end_pos(self):
if attr == "start_pos": """Returns end position respecting multiline tokens."""
self.start_pos_line = value[0] end_pos_line = self.start_pos_line
self.start_pos_col = value[1] lines = unicode(self).split('\n')
end_pos_line += len(lines) - 1
end_pos_col = self.start_pos_col
# Check for multiline token
if self.start_pos_line == end_pos_line:
end_pos_col += len(lines[-1])
else: else:
object.__setattr__(self, attr, value) end_pos_col = len(lines[-1])
return (end_pos_line, end_pos_col)
# Make cache footprint smaller for faster unpickling # Make cache footprint smaller for faster unpickling
def __getstate__(self): def __getstate__(self):
@@ -124,7 +122,7 @@ class Token(object):
) )
def __setstate__(self, state): def __setstate__(self, state):
self.token_type = state[0] self._token_type = state[0]
self.token = state[1] self._token = state[1]
self.start_pos_line = state[2] self._start_pos_line = state[2]
self.start_pos_col = state[3] self._start_pos_col = state[3]

25
test/test_token.py Normal file
View File

@@ -0,0 +1,25 @@
import jedi.parser as parser
try:
import unittest2 as unittest
except ImportError: # pragma: no cover
import unittest
class TokenTest(unittest.TestCase):
def test_end_pos_one_line(self):
parsed = parser.Parser('''
def testit():
a = "huhu"
''')
tok = parsed.top_module.subscopes[0].statements[0].token_list[2]
self.assertEqual(tok.end_pos, (3, 14))
def test_end_pos_multi_line(self):
parsed = parser.Parser('''
def testit():
a = """huhu
asdfasdf""" + "h"
''')
tok = parsed.top_module.subscopes[0].statements[0].token_list[2]
self.assertEqual(tok.end_pos, (4, 11))