VimPlug/jedi
1
0
Fork 1
mirror of https://github.com/davidhalter/jedi.git synced 2026-08-13 07:01:00 +08:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

replace parser tokenizer offset with line offset

Browse Source
This commit is contained in:
Dave Halter
2014-02-17 23:02:04 +01:00
parent 6952596117
commit 1b6df4602d
2 changed files with 7 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files
jedi/api/__init__.py
+3
View File
@@ -233,6 +233,9 @@ class Script(object):
stmt = r.module.statements[0] stmt = r.module.statements[0]
except IndexError: except IndexError:
raise NotFoundError() raise NotFoundError()
# Set the start_pos to a pseudo position, that doesn't exist but works
# perfectly well (for both completions in docstrings and statements).
stmt.start_pos = self._pos
stmt.parent = self._parser.user_scope() stmt.parent = self._parser.user_scope()
return stmt return stmt
jedi/parser/tokenize.py
+4 -19
View File
@@ -142,15 +142,14 @@ del _compile
tabsize = 8 tabsize = 8
def generate_tokens(readline, offset=(1, 0)): def generate_tokens(readline, line_offset=0):
lnum = offset[0] - 1 lnum = line_offset
parenlev = 0 parenlev = 0
continued = False continued = False
numchars = '0123456789' numchars = '0123456789'
contstr, needcont = '', 0 contstr, needcont = '', 0
contline = None contline = None
indents = [0] indents = [0]
first_pass = True
while True: # loop over lines in stream while True: # loop over lines in stream
try: try:
@@ -159,11 +158,7 @@ def generate_tokens(readline, offset=(1, 0)):
line = b'' line = b''
lnum += 1 lnum += 1
pos = 0
pos, max = 0, len(line) pos, max = 0, len(line)
if first_pass is True:
pos = offset[1]
first_pass = False
if contstr: # continued string if contstr: # continued string
if not line: if not line:
@@ -298,8 +293,7 @@ FLOWS = ['if', 'else', 'elif', 'while', 'with', 'try', 'except', 'finally']
class NoErrorTokenizer(object): class NoErrorTokenizer(object):
def __init__(self, readline, offset=(0, 0), is_fast_parser=False): def __init__(self, readline, offset=(0, 0), is_fast_parser=False):
self.readline = readline self.readline = readline
self.gen = generate_tokens(readline) self.gen = generate_tokens(readline, offset[0])
self.offset = offset
self.closed = False self.closed = False
self.is_first = True self.is_first = True
self.push_backs = [] self.push_backs = []
@@ -329,7 +323,7 @@ class NoErrorTokenizer(object):
self.last_previous = self.previous self.last_previous = self.previous
self.previous = self.current self.previous = self.current
self.current = next(self.gen) self.current = next(self.gen)
c = list(self.current) c = self.current
if c[0] == ENDMARKER: if c[0] == ENDMARKER:
self.current = self.previous self.current = self.previous
@@ -340,15 +334,6 @@ class NoErrorTokenizer(object):
# tokenize and therefore precise. # tokenize and therefore precise.
breaks = ['def', 'class', '@'] breaks = ['def', 'class', '@']
if self.is_first:
c[2] = self.offset[0] + c[2][0], self.offset[1] + c[2][1]
c[3] = self.offset[0] + c[3][0], self.offset[1] + c[3][1]
self.is_first = False
else:
c[2] = self.offset[0] + c[2][0], c[2][1]
c[3] = self.offset[0] + c[3][0], c[3][1]
self.current = c
def close(): def close():
if not self.first_stmt: if not self.first_stmt:
self.closed = True self.closed = True