VimPlug/jedi
1
0
Fork 1
mirror of https://github.com/davidhalter/jedi.git synced 2026-01-31 06:45:23 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

replace parser tokenizer offset with line offset

Browse Source
This commit is contained in:
Dave Halter
2014-02-17 23:02:04 +01:00
parent 6952596117
commit 1b6df4602d
2 changed files with 7 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
jedi/api/__init__.py
View File

@@ -233,6 +233,9 @@ class Script(object):
stmt = r.module.statements[0]
except IndexError:
raise NotFoundError()
# Set the start_pos to a pseudo position, that doesn't exist but works
# perfectly well (for both completions in docstrings and statements).
stmt.start_pos = self._pos
stmt.parent = self._parser.user_scope()
return stmt

23
jedi/parser/tokenize.py
View File

@@ -142,15 +142,14 @@ del _compile
tabsize = 8
def generate_tokens(readline, offset=(1, 0)):
lnum = offset[0] - 1
def generate_tokens(readline, line_offset=0):
lnum = line_offset
parenlev = 0
continued = False
numchars = '0123456789'
contstr, needcont = '', 0
contline = None
indents = [0]
first_pass = True
while True: # loop over lines in stream
try:
@@ -159,11 +158,7 @@ def generate_tokens(readline, offset=(1, 0)):
line = b''
lnum += 1
pos = 0
pos, max = 0, len(line)
if first_pass is True:
pos = offset[1]
first_pass = False
if contstr: # continued string
if not line:
@@ -298,8 +293,7 @@ FLOWS = ['if', 'else', 'elif', 'while', 'with', 'try', 'except', 'finally']
class NoErrorTokenizer(object):
def __init__(self, readline, offset=(0, 0), is_fast_parser=False):
self.readline = readline
self.gen = generate_tokens(readline)
self.offset = offset
self.gen = generate_tokens(readline, offset[0])
self.closed = False
self.is_first = True
self.push_backs = []
@@ -329,7 +323,7 @@ class NoErrorTokenizer(object):
self.last_previous = self.previous
self.previous = self.current
self.current = next(self.gen)
c = list(self.current)
c = self.current
if c[0] == ENDMARKER:
self.current = self.previous
@@ -340,15 +334,6 @@ class NoErrorTokenizer(object):
# tokenize and therefore precise.
breaks = ['def', 'class', '@']
if self.is_first:
c[2] = self.offset[0] + c[2][0], self.offset[1] + c[2][1]
c[3] = self.offset[0] + c[3][0], self.offset[1] + c[3][1]
self.is_first = False
else:
c[2] = self.offset[0] + c[2][0], c[2][1]
c[3] = self.offset[0] + c[3][0], c[3][1]
self.current = c
def close():
if not self.first_stmt:
self.closed = True