diff --git a/jedi/parser/__init__.py b/jedi/parser/__init__.py index ab921933..e4055d17 100644 --- a/jedi/parser/__init__.py +++ b/jedi/parser/__init__.py @@ -71,11 +71,9 @@ class Parser(object): # because of `self.module.used_names`. d.parent = self.module - if self._current[0] in (tokenize.NL, tokenize.NEWLINE): + if self._current[0] in (tokenize.NEWLINE,): # we added a newline before, so we need to "remove" it again. self.end_pos = self._gen.previous[2] - elif self._current[0] == tokenize.INDENT: - self.end_pos = self._gen.last_previous[2] self.start_pos = self.module.start_pos self.module.end_pos = self.end_pos @@ -403,7 +401,7 @@ class Parser(object): typ, tok, start_pos, end_pos = next(self._gen) # dedents shouldn't change positions self.start_pos = start_pos - if typ not in (tokenize.INDENT, tokenize.NEWLINE, tokenize.NL): + if typ not in (tokenize.INDENT, tokenize.NEWLINE): self.start_pos, self.end_pos = start_pos, end_pos self._current = typ, tok @@ -619,7 +617,7 @@ class Parser(object): self.freshscope = False else: if token_type not in [tokenize.COMMENT, tokenize.INDENT, - tokenize.NEWLINE, tokenize.NL]: + tokenize.NEWLINE]: debug.warning('Token not used: %s %s %s', tok, tokenize.tok_name[token_type], self.start_pos) continue @@ -655,7 +653,3 @@ class PushBackTokenizer(object): @property def previous(self): return self._tokenizer.previous - - @property - def last_previous(self): - return self._tokenizer.last_previous diff --git a/jedi/parser/tokenize.py b/jedi/parser/tokenize.py index 6b7edc29..9f049d2f 100644 --- a/jedi/parser/tokenize.py +++ b/jedi/parser/tokenize.py @@ -23,8 +23,6 @@ namechars = string.ascii_letters + '_' COMMENT = N_TOKENS tok_name[COMMENT] = 'COMMENT' -NL = N_TOKENS + 1 -tok_name[NL] = 'NL' ENCODING = N_TOKENS + 2 tok_name[ENCODING] = 'ENCODING' N_TOKENS += 3 @@ -215,11 +213,11 @@ def generate_tokens(readline, line_offset=0): nl_pos = pos + len(comment_token) yield TokenInfo(COMMENT, comment_token, (lnum, pos), (lnum, pos + len(comment_token))) - yield TokenInfo(NL, line[nl_pos:], (lnum, nl_pos), + yield TokenInfo(NEWLINE, line[nl_pos:], (lnum, nl_pos), (lnum, len(line))) else: yield TokenInfo( - (NL, COMMENT)[line[pos] == '#'], line[pos:], + (NEWLINE, COMMENT)[line[pos] == '#'], line[pos:], (lnum, pos), (lnum, len(line))) continue @@ -246,7 +244,7 @@ def generate_tokens(readline, line_offset=0): (initial == '.' and token != '.' and token != '...')): yield TokenInfo(NUMBER, token, spos, epos) elif initial in '\r\n': - yield TokenInfo(NL if parenlev > 0 else NEWLINE, + yield TokenInfo(NEWLINE, token, spos, epos) elif initial == '#': assert not token.endswith("\n") @@ -341,8 +339,8 @@ class NoErrorTokenizer(object): raise common.MultiLevelStopIteration() # ignore indents/comments if self.is_fast_parser \ - and self.previous[0] in (INDENT, NL, None, NEWLINE) \ - and c[0] not in (COMMENT, INDENT, NL, NEWLINE): + and self.previous[0] in (INDENT, None, NEWLINE) \ + and c[0] not in (COMMENT, INDENT, NEWLINE): # print c, tok_name[c[0]] tok = c[1]