VimPlug/jedi
1
0
Fork 1
mirror of https://github.com/davidhalter/jedi.git synced 2025-12-15 01:57:06 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

remove end_pos stuff from tokenizer, the tokens can do that themselves

Browse Source
This commit is contained in:
Dave Halter
2014-02-25 13:29:27 +01:00
parent 2252271bf5
commit 5b84f0b27f
3 changed files with 26 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
jedi/parser/__init__.py
View File

@@ -616,7 +616,7 @@ class PushBackTokenizer(object):
def __init__(self, tokenizer): def __init__(self, tokenizer):
self._tokenizer = tokenizer self._tokenizer = tokenizer
self._push_backs = [] self._push_backs = []
self.current = tokenize.TokenInfo(None, '', (0, 0), (0, 0)) self.current = tokenize.TokenInfo(None, '', (0, 0))
def push_last_back(self): def push_last_back(self):
self._push_backs.append(self.current) self._push_backs.append(self.current)

2
jedi/parser/fast.py
View File

@@ -398,7 +398,7 @@ class FastTokenizer(object):
self.closed = False self.closed = False
# fast parser options # fast parser options
self.current = self.previous = TokenInfo(None, '', (0, 0), (0, 0)) self.current = self.previous = TokenInfo(None, '', (0, 0))
self.in_flow = False self.in_flow = False
self.new_indent = False self.new_indent = False
self.parser_indent = self.old_parser_indent = 0 self.parser_indent = self.old_parser_indent = 0

65
jedi/parser/tokenize.py
View File

@@ -15,7 +15,6 @@ import re
from io import StringIO from io import StringIO
from token import (tok_name, N_TOKENS, ENDMARKER, STRING, NUMBER, NAME, OP, from token import (tok_name, N_TOKENS, ENDMARKER, STRING, NUMBER, NAME, OP,
ERRORTOKEN, NEWLINE) ERRORTOKEN, NEWLINE)
import collections
from jedi._compatibility import u, unicode from jedi._compatibility import u, unicode
@@ -36,13 +35,6 @@ ENCODING = N_TOKENS + 1
tok_name[ENCODING] = 'ENCODING' tok_name[ENCODING] = 'ENCODING'
class TokenInfo(collections.namedtuple('TokenInfo', 'type string start end')):
def __repr__(self):
annotated_type = '%d (%s)' % (self.type, tok_name[self.type])
return ('TokenInfo(type=%s, string=%r, start=%r, end=%r)' %
self._replace(type=annotated_type))
class TokenInfo(object): class TokenInfo(object):
"""The token object is an efficient representation of the structure """The token object is an efficient representation of the structure
(token_type, token, (start_pos_line, start_pos_col)). It has indexer (token_type, token, (start_pos_line, start_pos_col)). It has indexer
@@ -50,34 +42,33 @@ class TokenInfo(object):
structure. structure.
>>> tuple(TokenInfo(1, 'foo' ,(3,4))) >>> tuple(TokenInfo(1, 'foo' ,(3,4)))
(1, 'foo', (3, 4), None) (1, 'foo', (3, 4), (3, 7))
>>> str(TokenInfo(1, "test", (1, 1))) == "test" >>> str(TokenInfo(1, "test", (1, 1))) == "test"
True True
>>> repr(TokenInfo(1, "test", (1, 1))) >>> repr(TokenInfo(1, "test", (1, 1)))
"<TokenInfo: (1, 'test', (1, 1))>" "<TokenInfo: (1, 'test', (1, 1))>"
>>> TokenInfo(1, 2, (3, 4)).__getstate__() >>> TokenInfo(1, 'bar', (3, 4)).__getstate__()
(1, 2, 3, 4) (1, 'bar', 3, 4)
>>> a = TokenInfo(0, 0, (0, 0)) >>> a = TokenInfo(0, 'baz', (0, 0))
>>> a.__setstate__((1, 2, 3, 4)) >>> a.__setstate__((1, 'foo', 3, 4))
>>> a >>> a
<TokenInfo: (1, 2, (3, 4))> <TokenInfo: (1, 'foo', (3, 4))>
>>> a.start_pos >>> a.start_pos
(3, 4) (3, 4)
>>> a.string >>> a.string
2 'foo'
>>> a._start_pos_col >>> a._start_pos_col
4 4
>>> unicode(TokenInfo(1, u("😷"), (1 ,1))) + "p" == u("😷p") >>> unicode(TokenInfo(1, u("😷"), (1 ,1))) + "p" == u("😷p")
True True
""" """
__slots__ = ("type", "string", "_start_pos_line", "_start_pos_col", "end_pos") __slots__ = ("type", "string", "_start_pos_line", "_start_pos_col")
def __init__(self, type, string, start_pos, end_pos=None): def __init__(self, type, string, start_pos):
self.type = type self.type = type
self.string = string self.string = string
self._start_pos_line = start_pos[0] self._start_pos_line = start_pos[0]
self._start_pos_col = start_pos[1] self._start_pos_col = start_pos[1]
self.end_pos = end_pos
def __repr__(self): def __repr__(self):
return "<%s: %s>" % (type(self).__name__, tuple(self)[:3]) return "<%s: %s>" % (type(self).__name__, tuple(self)[:3])
@@ -120,13 +111,6 @@ class TokenInfo(object):
@property @property
def end(self): def end(self):
if self.end_pos is not None and self._end != self.end_pos:
print(self.end_pos, self._end, repr(self.string))
assert False
return self.end_pos
@property
def _end(self):
"""Returns end position respecting multiline tokens.""" """Returns end position respecting multiline tokens."""
end_pos_line = self._start_pos_line end_pos_line = self._start_pos_line
lines = self.string.split('\n') lines = self.string.split('\n')
@@ -284,7 +268,7 @@ def generate_tokens(readline, line_offset=0):
line = readline() # readline returns empty if it's finished. See StringIO line = readline() # readline returns empty if it's finished. See StringIO
if not line: if not line:
if contstr: if contstr:
yield TokenInfo(ERRORTOKEN, contstr, contstr_start, (lnum, pos)) yield TokenInfo(ERRORTOKEN, contstr, contstr_start)
break break
lnum += 1 lnum += 1
@@ -293,8 +277,8 @@ def generate_tokens(readline, line_offset=0):
if contstr: # continued string if contstr: # continued string
endmatch = endprog.match(line) endmatch = endprog.match(line)
if endmatch: if endmatch:
pos = end = endmatch.end(0) pos = endmatch.end(0)
yield TokenInfo(STRING, contstr + line[:end], contstr_start, (lnum, end)) yield TokenInfo(STRING, contstr + line[:pos], contstr_start)
contstr = '' contstr = ''
contline = None contline = None
else: else:
@@ -305,30 +289,29 @@ def generate_tokens(readline, line_offset=0):
while pos < max: while pos < max:
pseudomatch = pseudoprog.match(line, pos) pseudomatch = pseudoprog.match(line, pos)
if not pseudomatch: # scan for tokens if not pseudomatch: # scan for tokens
yield TokenInfo(ERRORTOKEN, line[pos], yield TokenInfo(ERRORTOKEN, line[pos], (lnum, pos))
(lnum, pos), (lnum, pos + 1))
pos += 1 pos += 1
continue continue
start, end = pseudomatch.span(1) start, pos = pseudomatch.span(1)
spos, epos, pos = (lnum, start), (lnum, end), end spos = (lnum, start)
token, initial = line[start:end], line[start] token, initial = line[start:pos], line[start]
if (initial in numchars or # ordinary number if (initial in numchars or # ordinary number
(initial == '.' and token != '.' and token != '...')): (initial == '.' and token != '.' and token != '...')):
yield TokenInfo(NUMBER, token, spos, epos) yield TokenInfo(NUMBER, token, spos)
elif initial in '\r\n': elif initial in '\r\n':
yield TokenInfo(NEWLINE, token, spos, epos) yield TokenInfo(NEWLINE, token, spos)
elif initial == '#': elif initial == '#':
assert not token.endswith("\n") assert not token.endswith("\n")
yield TokenInfo(COMMENT, token, spos, epos) yield TokenInfo(COMMENT, token, spos)
elif token in triple_quoted: elif token in triple_quoted:
endprog = endprogs[token] endprog = endprogs[token]
endmatch = endprog.match(line, pos) endmatch = endprog.match(line, pos)
if endmatch: # all on one line if endmatch: # all on one line
pos = endmatch.end(0) pos = endmatch.end(0)
token = line[start:pos] token = line[start:pos]
yield TokenInfo(STRING, token, spos, (lnum, pos)) yield TokenInfo(STRING, token, spos)
else: else:
contstr_start = (lnum, start) # multiple lines contstr_start = (lnum, start) # multiple lines
contstr = line[start:] contstr = line[start:]
@@ -345,12 +328,12 @@ def generate_tokens(readline, line_offset=0):
contline = line contline = line
break break
else: # ordinary string else: # ordinary string
yield TokenInfo(STRING, token, spos, epos) yield TokenInfo(STRING, token, spos)
elif initial in namechars: # ordinary name elif initial in namechars: # ordinary name
yield TokenInfo(NAME, token, spos, epos) yield TokenInfo(NAME, token, spos)
elif initial == '\\' and line[start:] == '\\\n': # continued stmt elif initial == '\\' and line[start:] == '\\\n': # continued stmt
continue continue
else: else:
yield TokenInfo(OP, token, spos, epos) yield TokenInfo(OP, token, spos)
yield TokenInfo(ENDMARKER, '', (lnum, 0), (lnum, 0)) yield TokenInfo(ENDMARKER, '', (lnum, 0))