Make the tokenizer a generator.

2017-03-09 18:53:09 +01:00
parent 989e4bac89
commit c7a74e6d1c
4 changed files with 41 additions and 52 deletions
--- a/jedi/parser/diff.py
+++ b/jedi/parser/diff.py
@@ -356,8 +356,7 @@ class DiffParser(object):
        is_first_token = True
        omitted_first_indent = False
        indents = []
-        l = iter(lines)
-        tokens = generate_tokens(lambda: next(l, ''), use_exact_op_types=True)
+        tokens = generate_tokens(lines, use_exact_op_types=True)
        stack = self._active_parser.pgen_parser.stack
        for typ, string, start_pos, prefix in tokens:
            start_pos = start_pos[0] + line_offset, start_pos[1]