diff --git a/jedi/parser/__init__.py b/jedi/parser/__init__.py index 12495722..95fd80dc 100644 --- a/jedi/parser/__init__.py +++ b/jedi/parser/__init__.py @@ -38,7 +38,7 @@ class Parser(object): :param top_module: Use this module as a parent instead of `self.module`. """ def __init__(self, source, module_path=None, no_docstr=False, - tokenizer=None, top_module=None, offset=0, is_fast=False): + tokenizer=None, top_module=None, offset=0): self.no_docstr = no_docstr self.start_pos = self.end_pos = 1 + offset, 0 @@ -642,12 +642,6 @@ class PushBackTokenizer(object): return self._push_backs.pop(0) self.current = next(self._tokenizer) - - def close(): - if not self.first_stmt: - self.closed = True - raise common.MultiLevelStopIteration() - # ignore indents/comments return self.current @property diff --git a/jedi/parser/fast.py b/jedi/parser/fast.py index c5ff70ea..a8faad49 100644 --- a/jedi/parser/fast.py +++ b/jedi/parser/fast.py @@ -364,10 +364,10 @@ class FastParser(use_metaclass(CachedFastParser)): if nodes[index].code != code: raise ValueError() except ValueError: - tokenizer = FastTokenizer(parser_code, line_offset, True) + tokenizer = FastTokenizer(parser_code, line_offset) p = Parser(parser_code, self.module_path, tokenizer=tokenizer, top_module=self.module, no_docstr=no_docstr, - is_fast=True, offset=line_offset) + offset=line_offset) p.module.parent = self.module else: if nodes[index] != self.current_node: @@ -390,13 +390,12 @@ class FastTokenizer(object): """ Breaks when certain conditions are met, i.e. a new function or class opens. """ - def __init__(self, source, line_offset=0, is_fast_parser=False): + def __init__(self, source, line_offset=0): self.source = source self.gen = source_tokens(source, line_offset) self.closed = False # fast parser options - self.is_fast_parser = is_fast_parser self.current = self.previous = TokenInfo(None, None, (0, 0), (0, 0)) self.in_flow = False self.new_indent = False @@ -428,11 +427,8 @@ class FastTokenizer(object): self.closed = True raise common.MultiLevelStopIteration() # ignore comments/ newlines - if self.is_fast_parser \ - and self.previous[0] in (None, NEWLINE) \ - and current[0] not in (COMMENT, NEWLINE): + if self.previous[0] in (None, NEWLINE) and current[0] not in (COMMENT, NEWLINE): # print c, tok_name[c[0]] - tok = current[1] indent = current[2][1] if indent < self.parser_indent: # -> dedent diff --git a/jedi/parser/tokenize.py b/jedi/parser/tokenize.py index 44cb22d0..03a66e29 100644 --- a/jedi/parser/tokenize.py +++ b/jedi/parser/tokenize.py @@ -148,6 +148,7 @@ tabsize = 8 def source_tokens(source, line_offset=0): + """Generate tokens from a the source code (string).""" source = source + '\n' # end with \n, because the parser needs it readline = StringIO(source).readline return generate_tokens(readline, line_offset)