moved NoErrorTokenizer to fast.FastTokenizer

jedi/parser/tokenize.py

@@ -19,6 +19,12 @@ cookie_re = re.compile("coding[:=]\s*([-\w.]+)")
 
 from jedi import common
 
+
+# From here on we have custom stuff (everything before was originally Python
+# internal code).
+FLOWS = ['if', 'else', 'elif', 'while', 'with', 'try', 'except', 'finally']
+
+
 namechars = string.ascii_letters + '_'
 
 
@@ -230,82 +236,3 @@ def generate_tokens(readline, line_offset=0):
                 yield TokenInfo(OP, token, spos, epos)
 
     yield TokenInfo(ENDMARKER, '', (lnum, 0), (lnum, 0))
-
-
-# From here on we have custom stuff (everything before was originally Python
-# internal code).
-FLOWS = ['if', 'else', 'elif', 'while', 'with', 'try', 'except', 'finally']
-
-
-class NoErrorTokenizer(object):
-    def __init__(self, source, line_offset=0, is_fast_parser=False):
-        self.source = source
-        self.gen = source_tokens(source, line_offset)
-        self.closed = False
-
-        # fast parser options
-        self.is_fast_parser = is_fast_parser
-        self.current = self.previous = TokenInfo(None, None, (0, 0), (0, 0))
-        self.in_flow = False
-        self.new_indent = False
-        self.parser_indent = self.old_parser_indent = 0
-        self.is_decorator = False
-        self.first_stmt = True
-
-    def next(self):
-        """ Python 2 Compatibility """
-        return self.__next__()
-
-    def __next__(self):
-        if self.closed:
-            raise common.MultiLevelStopIteration()
-
-        current = next(self.gen)
-        if current[0] == ENDMARKER:
-            raise common.MultiLevelStopIteration()
-
-        self.previous = self.current
-        self.current = current
-
-        # this is exactly the same check as in fast_parser, but this time with
-        # tokenize and therefore precise.
-        breaks = ['def', 'class', '@']
-
-        def close():
-            if not self.first_stmt:
-                self.closed = True
-                raise common.MultiLevelStopIteration()
-        # ignore comments/ newlines
-        if self.is_fast_parser \
-                and self.previous[0] in (None, NEWLINE) \
-                and current[0] not in (COMMENT, NEWLINE):
-            # print c, tok_name[c[0]]
-
-            tok = current[1]
-            indent = current[2][1]
-            if indent < self.parser_indent:  # -> dedent
-                self.parser_indent = indent
-                self.new_indent = False
-                if not self.in_flow or indent < self.old_parser_indent:
-                    close()
-                self.in_flow = False
-            elif self.new_indent:
-                self.parser_indent = indent
-                self.new_indent = False
-
-            if not self.in_flow:
-                if tok in FLOWS or tok in breaks:
-                    self.in_flow = tok in FLOWS
-                    if not self.is_decorator and not self.in_flow:
-                        close()
-                    self.is_decorator = '@' == tok
-                    if not self.is_decorator:
-                        self.old_parser_indent = self.parser_indent
-                        self.parser_indent += 1  # new scope: must be higher
-                        self.new_indent = True
-
-            if tok != '@':
-                if self.first_stmt and not self.new_indent:
-                    self.parser_indent = indent
-                self.first_stmt = False
-        return current