diff --git a/jedi/evaluate/finder.py b/jedi/evaluate/finder.py index 3b128bdf..e75e602b 100644 --- a/jedi/evaluate/finder.py +++ b/jedi/evaluate/finder.py @@ -381,7 +381,7 @@ def get_names_of_scope(evaluator, scope, position=None, star_search=True, includ ... ''')) >>> scope = parser.module.subscopes[0] >>> scope - + `get_names_of_scope` is a generator. First it yields names from most inner scope. @@ -389,13 +389,13 @@ def get_names_of_scope(evaluator, scope, position=None, star_search=True, includ >>> from jedi.evaluate import Evaluator >>> pairs = list(get_names_of_scope(Evaluator(), scope)) >>> pairs[0] - (, []) + (, []) Then it yield the names from one level outer scope. For this example, this is the most outer scope. >>> pairs[1] - (, [, ]) + (, [, ]) Finally, it yields names from builtin, if `include_builtin` is true (default). diff --git a/jedi/parser/__init__.py b/jedi/parser/__init__.py index c4e23ceb..12495722 100644 --- a/jedi/parser/__init__.py +++ b/jedi/parser/__init__.py @@ -47,8 +47,7 @@ class Parser(object): self._scope = self.module self._current = (None, None) - tokenizer = tokenizer or tokenize.NoErrorTokenizer(source) - tokenizer = tokenize.NoErrorTokenizer(source, offset, is_fast) + tokenizer = tokenizer or tokenize.source_tokens(source) self._gen = PushBackTokenizer(tokenizer) self._top_module = top_module or self.module try: @@ -59,10 +58,11 @@ class Parser(object): # sometimes StopIteration isn't catched. Just ignore it. # on finish, set end_pos correctly - s = self._scope - while s is not None: - s.end_pos = self.end_pos - s = s.parent + pass + s = self._scope + while s is not None: + s.end_pos = self.end_pos + s = s.parent # clean up unused decorators for d in self._decorators: @@ -71,6 +71,8 @@ class Parser(object): d.parent = self.module if self._current[0] in (tokenize.NEWLINE,): + # This case is only relevant with the FastTokenizer, because + # otherwise there's always an EndMarker. # we added a newline before, so we need to "remove" it again. self.end_pos = self._gen.previous[2] @@ -626,7 +628,7 @@ class PushBackTokenizer(object): def __init__(self, tokenizer): self._tokenizer = tokenizer self._push_backs = [] - self.current = [None, None, (0, 0), (0, 0), ''] + self.current = tokenize.TokenInfo(None, None, (0, 0), (0, 0)) def push_last_back(self): self._push_backs.append(self.current) diff --git a/test/test_parser/test_parsing.py b/test/test_parser/test_parsing.py index 7dc347f1..85323ccd 100644 --- a/test/test_parser/test_parsing.py +++ b/test/test_parser/test_parsing.py @@ -2,6 +2,7 @@ from jedi._compatibility import u from jedi.parser import Parser from jedi.parser.user_context import UserContextParser from jedi.parser import representation as pr +from textwrap import dedent def test_user_statement_on_import(): @@ -95,3 +96,15 @@ def test_module(): assert str(name) == '' assert name.start_pos == (0, 0) assert name.end_pos == (0, 0) + + +def test_end_pos(): + s = u(dedent(''' + x = ['a', 'b', 'c'] + def func(): + y = None + ''')) + parser = Parser(s) + scope = parser.module.subscopes[0] + assert scope.start_pos == (3, 0) + assert scope.end_pos == (5, 0)