VimPlug/jedi-fork
1
0
Fork 0
forked from VimPlug/jedi
Code Pull Requests Activity

fix for problems with incomplete one liner string literals, after a start of an incomplete string literal the whole line should be seen as an error token

Browse Source
This commit is contained in:
Dave Halter
2014-04-19 13:56:04 +02:00
parent 54568c1868
commit bb6874bc7c
2 changed files with 14 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
jedi/parser/tokenize.py
View File

@@ -42,13 +42,13 @@ class Token(object):
structure. structure.
>>> repr(Token(1, "test", (1, 1))) >>> repr(Token(1, "test", (1, 1)))
"<Token: (1, 'test', (1, 1))>" "<Token: ('NAME', 'test', (1, 1))>"
>>> Token(1, 'bar', (3, 4)).__getstate__() >>> Token(1, 'bar', (3, 4)).__getstate__()
(1, 'bar', 3, 4) (1, 'bar', 3, 4)
>>> a = Token(0, 'baz', (0, 0)) >>> a = Token(0, 'baz', (0, 0))
>>> a.__setstate__((1, 'foo', 3, 4)) >>> a.__setstate__((1, 'foo', 3, 4))
>>> a >>> a
<Token: (1, 'foo', (3, 4))> <Token: ('NAME', 'foo', (3, 4))>
>>> a.start_pos >>> a.start_pos
(3, 4) (3, 4)
>>> a.string >>> a.string
@@ -67,7 +67,8 @@ class Token(object):
self._start_pos_col = start_pos[1] self._start_pos_col = start_pos[1]
def __repr__(self): def __repr__(self):
content = self.type, self.string, (self._start_pos_line, self._start_pos_col) typ = tok_name[self.type]
content = typ, self.string, (self._start_pos_line, self._start_pos_col)
return "<%s: %s>" % (type(self).__name__, content) return "<%s: %s>" % (type(self).__name__, content)
@property @property
@@ -249,7 +250,12 @@ def generate_tokens(readline, line_offset=0):
while pos < max: while pos < max:
pseudomatch = pseudoprog.match(line, pos) pseudomatch = pseudoprog.match(line, pos)
if not pseudomatch: # scan for tokens if not pseudomatch: # scan for tokens
yield Token(ERRORTOKEN, line[pos], (lnum, pos)) txt = line[pos]
if line[pos] in '"\'':
# If a literal starts but doesn't end the whole rest of the
# line is an error token.
txt = txt = line[pos:]
yield Token(ERRORTOKEN, txt, (lnum, pos))
pos += 1 pos += 1
continue continue

6
test/test_api/test_call_signatures.py
View File

@@ -162,8 +162,10 @@ class TestCallSignatures(TestCase):
assert x == ['*args'] assert x == ['*args']
def test_additional_brackets(self): def test_additional_brackets(self):
s = 'str((' self._run('str((', 'str', 0)
self._run(s, 'str', 0)
def test_unterminated_strings(self):
self._run('str(";', 'str', 0)
class TestParams(TestCase): class TestParams(TestCase):