diff --git a/jedi/parser/tokenize.py b/jedi/parser/tokenize.py index 3b040f3a..e01b40bf 100644 --- a/jedi/parser/tokenize.py +++ b/jedi/parser/tokenize.py @@ -42,13 +42,13 @@ class Token(object): structure. >>> repr(Token(1, "test", (1, 1))) - "" + "" >>> Token(1, 'bar', (3, 4)).__getstate__() (1, 'bar', 3, 4) >>> a = Token(0, 'baz', (0, 0)) >>> a.__setstate__((1, 'foo', 3, 4)) >>> a - + >>> a.start_pos (3, 4) >>> a.string @@ -67,7 +67,8 @@ class Token(object): self._start_pos_col = start_pos[1] def __repr__(self): - content = self.type, self.string, (self._start_pos_line, self._start_pos_col) + typ = tok_name[self.type] + content = typ, self.string, (self._start_pos_line, self._start_pos_col) return "<%s: %s>" % (type(self).__name__, content) @property @@ -249,7 +250,12 @@ def generate_tokens(readline, line_offset=0): while pos < max: pseudomatch = pseudoprog.match(line, pos) if not pseudomatch: # scan for tokens - yield Token(ERRORTOKEN, line[pos], (lnum, pos)) + txt = line[pos] + if line[pos] in '"\'': + # If a literal starts but doesn't end the whole rest of the + # line is an error token. + txt = txt = line[pos:] + yield Token(ERRORTOKEN, txt, (lnum, pos)) pos += 1 continue diff --git a/test/test_api/test_call_signatures.py b/test/test_api/test_call_signatures.py index b7667c3d..4c7cd78c 100644 --- a/test/test_api/test_call_signatures.py +++ b/test/test_api/test_call_signatures.py @@ -162,8 +162,10 @@ class TestCallSignatures(TestCase): assert x == ['*args'] def test_additional_brackets(self): - s = 'str((' - self._run(s, 'str', 0) + self._run('str((', 'str', 0) + + def test_unterminated_strings(self): + self._run('str(";', 'str', 0) class TestParams(TestCase):