The new tokenizer is more or less working now. Indents are calculated as they should

jedi/parser/pgen2/parse.py

@@ -96,9 +96,32 @@ class Parser(object):
         self.stack = [stackentry]
         self.rootnode = None
         self.error_recovery = error_recovery
+        indent_errors = []  # TODO generate those.
 
-    def parse(self, tokens):
-        for type, value, prefix, start_pos in tokens:
+    def tokenize(self, tokenizer):
+        """
+        This is not a real tokenizer, but it adds indents. You could hand the
+        parse function a normal tokenizer (e.g. the lib2to3 one). But if we use
+        the parser stack we are able to do error recovery from wrong indents.
+        """
+        indents = [0]
+        new_line = False
+        for type, value, prefix, start_pos in tokenizer:
+            if type == token.NEWLINE:
+                new_line = True
+            elif new_line:
+                indent = start_pos[1]
+                if indent > indents[-1]:
+                    yield token.INDENT, '', '', start_pos
+                    indents.append(indent)
+                while indent < indents[-1]:
+                    yield token.DEDENT, '', '', start_pos
+                    indents.pop()
+                new_line = False
+            yield type, value, prefix, start_pos
+
+    def parse(self, tokenizer):
+        for type, value, prefix, start_pos in tokenizer:
             if self.addtoken(type, value, prefix, start_pos):
                 break
         else: