VimPlug/jedi
1
0
Fork 1
mirror of https://github.com/davidhalter/jedi.git synced 2025-12-10 15:51:51 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

restructure user_context reverse tokenizer to remove INDENT tokens in the future

Browse Source
This commit is contained in:
Dave Halter
2014-02-21 15:30:46 +01:00
parent 9c3b7b9c46
commit fe02088dd7
3 changed files with 15 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
jedi/parser/tokenize.py
View File

@@ -211,7 +211,8 @@ def generate_tokens(readline, line_offset=0):
if column > indents[-1]: # count indents or dedents if column > indents[-1]: # count indents or dedents
indents.append(column) indents.append(column)
yield TokenInfo(INDENT, line[:pos], (lnum, 0), (lnum, pos)) #print repr(line), lnum
#yield TokenInfo(INDENT, line[:pos], (lnum, 0), (lnum, pos))
while column < indents[-1]: while column < indents[-1]:
indents = indents[:-1] indents = indents[:-1]

15
jedi/parser/user_context.py
View File

@@ -30,11 +30,14 @@ class UserContext(object):
return path return path
def _calc_path_until_cursor(self, start_pos=None): def _calc_path_until_cursor(self, start_pos=None):
"""
Something like a reverse tokenizer that tokenizes the reversed strings.
"""
def fetch_line(): def fetch_line():
if self._is_first: if self._is_first:
self._is_first = False self._is_first = False
self._line_length = self._column_temp self._line_length = self._column_temp
line = self._first_line line = first_line
else: else:
line = self.get_line(self._line_temp) line = self.get_line(self._line_temp)
self._line_length = len(line) self._line_length = len(line)
@@ -53,7 +56,7 @@ class UserContext(object):
self._is_first = True self._is_first = True
self._line_temp, self._column_temp = start_cursor = start_pos self._line_temp, self._column_temp = start_cursor = start_pos
self._first_line = self.get_line(self._line_temp)[:self._column_temp] first_line = self.get_line(self._line_temp)[:self._column_temp]
open_brackets = ['(', '[', '{'] open_brackets = ['(', '[', '{']
close_brackets = [')', ']', '}'] close_brackets = [')', ']', '}']
@@ -63,7 +66,13 @@ class UserContext(object):
level = 0 level = 0
force_point = False force_point = False
last_type = None last_type = None
is_first = True
for token_type, tok, start, end in gen: for token_type, tok, start, end in gen:
if is_first:
if start != (1, 0): # whitespace is not a path
return '', start_cursor
is_first = False
# print 'tok', token_type, tok, force_point # print 'tok', token_type, tok, force_point
if last_type == token_type == tokenize.NAME: if last_type == token_type == tokenize.NAME:
string += ' ' string += ' '
@@ -94,7 +103,7 @@ class UserContext(object):
x = start_pos[0] - end[0] + 1 x = start_pos[0] - end[0] + 1
l = self.get_line(x) l = self.get_line(x)
l = self._first_line if x == start_pos[0] else l l = first_line if x == start_pos[0] else l
start_cursor = x, len(l) - end[1] start_cursor = x, len(l) - end[1]
self._column_temp = self._line_length - end[1] self._column_temp = self._line_length - end[1]
string += tok string += tok

2
test/test_integration_import.py
View File

@@ -63,8 +63,8 @@ def test_after_from():
completions = Script(source, column=column).completions() completions = Script(source, column=column).completions()
assert [c.name for c in completions] == result assert [c.name for c in completions] == result
check('from os ', ['import'])
check('\nfrom os ', ['import']) check('\nfrom os ', ['import'])
check('from os ', ['import'])
check('\nfrom os import whatever', ['import'], len('from os im')) check('\nfrom os import whatever', ['import'], len('from os im'))
check('from os\\\n', ['import']) check('from os\\\n', ['import'])