Implement binary search for get_leaf_for_position. This makes it a lot faster.

jedi/parser/__init__.py

@@ -223,20 +223,11 @@ class Parser(object):
                 newline = endmarker.get_previous_leaf()
             except IndexError:
                 return  # This means that the parser is empty.
-            while True:
-                if newline.value == '':
-                    # Must be a DEDENT, just continue.
-                    try:
-                        newline = newline.get_previous_leaf()
-                    except IndexError:
-                        # If there's a statement that fails to be parsed, there
-                        # will be no previous leaf. So just ignore it.
-                        break
-                else:
+
             assert newline.value.endswith('\n')
             newline.value = newline.value[:-1]
-                    endmarker.start_pos = newline.start_pos
-                    break
+            endmarker.start_pos = \
+                newline.start_pos[0], newline.start_pos[1] + len(newline.value)
 
 
 class ParserWithRecovery(Parser):

jedi/parser/tokenize.py

@@ -258,6 +258,8 @@ def generate_tokens(readline, use_exact_op_types=False):
                     # If a literal starts but doesn't end the whole rest of the
                     # line is an error token.
                     txt = line[pos:]
+                if txt.endswith('\n'):
+                    new_line = True
                 yield TokenInfo(ERRORTOKEN, txt, (lnum, pos), prefix)
                 break
 

jedi/parser/tree.py

@@ -543,19 +543,29 @@ class BaseNode(Base):
         return None
 
     def get_leaf_for_position(self, position, include_prefixes=False):
-        for c in self.children:
-            if include_prefixes:
-                start_pos = c.get_start_pos_of_prefix()
-            else:
-                start_pos = c.start_pos
-
-            if start_pos <= position <= c.end_pos:
-                try:
-                    return c.get_leaf_for_position(position, include_prefixes)
-                except AttributeError:
-                    return c
-
+        def binary_search(lower, upper):
+            if lower == upper:
+                element = self.children[lower]
+                if not include_prefixes and position < element.start_pos:
+                    # We're on a prefix.
                     return None
+                # In case we have prefixes, a leaf always matches
+                try:
+                    return element.get_leaf_for_position(position, include_prefixes)
+                except AttributeError:
+                    return element
+
+
+            index = int((lower + upper) / 2)
+            element = self.children[index]
+            if position <= element.end_pos:
+                return binary_search(lower, index)
+            else:
+                return binary_search(index + 1, upper)
+
+        if not ((1, 0) <= position <= self.children[-1].end_pos):
+            raise ValueError('Please provide a position that exists within this node.')
+        return binary_search(0, len(self.children) - 1)
 
     @Python3Method
     def get_statement_for_position(self, pos):