From c26ae3c00d951ec2aca1b3141bd6ca988cdacead Mon Sep 17 00:00:00 2001
From: Dave Halter <davidhalter88@gmail.com>
Date: Thu, 20 Feb 2014 01:21:20 +0100
Subject: [PATCH] ignore dedents in tokenizer

---
 jedi/parser/__init__.py | 16 +++-------------
 jedi/parser/tokenize.py | 12 ++++++------
 2 files changed, 9 insertions(+), 19 deletions(-)

diff --git a/jedi/parser/__init__.py b/jedi/parser/__init__.py
index f69cf077..ab921933 100644
--- a/jedi/parser/__init__.py
+++ b/jedi/parser/__init__.py
@@ -402,10 +402,9 @@ class Parser(object):
         """ Generate the next tokenize pattern. """
         typ, tok, start_pos, end_pos = next(self._gen)
         # dedents shouldn't change positions
-        if typ != tokenize.DEDENT:
-            self.start_pos = start_pos
-            if typ not in (tokenize.INDENT, tokenize.NEWLINE, tokenize.NL):
-                self.start_pos, self.end_pos = start_pos, end_pos
+        self.start_pos = start_pos
+        if typ not in (tokenize.INDENT, tokenize.NEWLINE, tokenize.NL):
+            self.start_pos, self.end_pos = start_pos, end_pos
 
         self._current = typ, tok
         return self._current
@@ -433,15 +432,6 @@ class Parser(object):
             # debug.dbg('main: tok=[%s] type=[%s] indent=[%s]', \
             #           tok, tokenize.tok_name[token_type], start_position[0])
 
-            while token_type == tokenize.DEDENT and self._scope != self.module:
-                token_type, tok = self.next()
-                if self.start_pos[1] <= self._scope.start_pos[1]:
-                    self._scope.end_pos = self.start_pos
-                    self._scope = self._scope.parent
-                    if isinstance(self._scope, pr.Module) \
-                            and not isinstance(self._scope, pr.SubModule):
-                        self._scope = self.module
-
             # check again for unindented stuff. this is true for syntax
             # errors. only check for names, because thats relevant here. If
             # some docstrings are not indented, I don't care.
diff --git a/jedi/parser/tokenize.py b/jedi/parser/tokenize.py
index d480705c..6b7edc29 100644
--- a/jedi/parser/tokenize.py
+++ b/jedi/parser/tokenize.py
@@ -150,7 +150,10 @@ def source_tokens(source, line_offset=0):
 
 
 def generate_tokens(readline, line_offset=0):
-    """The original stdlib Python version with minor modifications"""
+    """
+    The original stdlib Python version with minor modifications.
+    Modified to not care about dedents.
+    """
     lnum = line_offset
     parenlev = 0
     continued = False
@@ -225,7 +228,6 @@ def generate_tokens(readline, line_offset=0):
                 yield TokenInfo(INDENT, line[:pos], (lnum, 0), (lnum, pos))
             while column < indents[-1]:
                 indents = indents[:-1]
-                yield TokenInfo(DEDENT, '', (lnum, pos), (lnum, pos))
 
         else:                                  # continued statement
             if not line:
@@ -288,8 +290,6 @@ def generate_tokens(readline, line_offset=0):
                                (lnum, pos), (lnum, pos + 1))
                 pos += 1
 
-    for indent in indents[1:]:                 # pop remaining indent levels
-        yield TokenInfo(DEDENT, '', (lnum, 0), (lnum, 0))
     yield TokenInfo(ENDMARKER, '', (lnum, 0), (lnum, 0))
 
 
@@ -341,8 +341,8 @@ class NoErrorTokenizer(object):
                 raise common.MultiLevelStopIteration()
         # ignore indents/comments
         if self.is_fast_parser \
-                and self.previous[0] in (INDENT, NL, None, NEWLINE, DEDENT) \
-                and c[0] not in (COMMENT, INDENT, NL, NEWLINE, DEDENT):
+                and self.previous[0] in (INDENT, NL, None, NEWLINE) \
+                and c[0] not in (COMMENT, INDENT, NL, NEWLINE):
             # print c, tok_name[c[0]]
 
             tok = c[1]