From 184e90a3a0c45193d11bee28c1b831ffbcfd288c Mon Sep 17 00:00:00 2001
From: Dave Halter <davidhalter88@gmail.com>
Date: Fri, 21 Feb 2014 00:13:23 +0100
Subject: [PATCH] removed some line checks from tokenizer

---
 jedi/parser/tokenize.py | 12 +-----------
 1 file changed, 1 insertion(+), 11 deletions(-)

diff --git a/jedi/parser/tokenize.py b/jedi/parser/tokenize.py
index b9c8e982..405647b9 100644
--- a/jedi/parser/tokenize.py
+++ b/jedi/parser/tokenize.py
@@ -165,17 +165,12 @@ def generate_tokens(readline, line_offset=0):
         except StopIteration:
             if contstr:
                 yield TokenInfo(ERRORTOKEN, contstr, strstart, (lnum, pos))
-                contstr, needcont = '', False
-                contline = None
-            line = b''
+            break
 
         lnum += 1
         pos, max = 0, len(line)
 
         if contstr:                            # continued string
-            if not line:
-                # multiline string has not been finished
-                break
             endmatch = endprog.match(line)
             if endmatch:
                 pos = end = endmatch.end(0)
@@ -195,8 +190,6 @@ def generate_tokens(readline, line_offset=0):
                 continue
 
         elif not continued:  # new statement
-            if not line:
-                break
             column = 0
             while pos < max:                   # measure leading whitespace
                 if line[pos] == ' ':
@@ -230,9 +223,6 @@ def generate_tokens(readline, line_offset=0):
                 indents = indents[:-1]
 
         else:                                  # continued statement
-            if not line:
-                # basically a statement has not been finished here.
-                break
             continued = False
 
         while pos < max: